diff --git a/CHANGELOG.md b/CHANGELOG.md index b71eba1eaa..582a45f492 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,16 @@ ## [Unreleased] -- No changes yet. +- Add support for `yaml` format. All commands that take image inputs, output images, + or convert between message formats, now take `yaml` as a format, in addition to + the existing `binpb` and `txtpb` formats. Some examples: + - `buf build -o image.yaml` + - `buf ls-files image.yaml` + - `buf convert --type foo.Bar --from input.binpb --to output.yaml` +- The `yaml` and `json` formats now accept two new options: `use_proto_names` and + `use_enum_numbers`. This affects output serialization. Some examples: + - `buf convert --type foo.Bar --from input.binpb --to output.yaml#use_proto_names=true` + - `buf convert --type foo.Bar --from input.binpb --to -#format=yaml,use_enum_numbers=true` ## [v1.28.1] - 2023-11-15 diff --git a/go.mod b/go.mod index a6bd6fc177..8f5be63974 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( connectrpc.com/otelconnect v0.6.0 github.com/bufbuild/protocompile v0.6.1-0.20231108163138-146b831231f7 github.com/bufbuild/protovalidate-go v0.4.1 + github.com/bufbuild/protoyaml-go v0.1.6 github.com/docker/docker v24.0.7+incompatible github.com/go-chi/chi/v5 v5.0.10 github.com/gofrs/flock v0.8.1 diff --git a/go.sum b/go.sum index 5509cbfa11..6adaf8682b 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/bufbuild/protocompile v0.6.1-0.20231108163138-146b831231f7 h1:1pUks8V github.com/bufbuild/protocompile v0.6.1-0.20231108163138-146b831231f7/go.mod h1:9N39DyRmxAF5+5AjqXQKV6hyWDI0EeoX4TRMix2ZnPE= github.com/bufbuild/protovalidate-go v0.4.1 h1:ye/8S72WbEklCeltPkSEeT8Eu1A7P/gmMsmapkwqTFk= github.com/bufbuild/protovalidate-go v0.4.1/go.mod h1:+p5FXfOjSEgLz5WBDTOMPMdQPXqALEERbJZU7huDCtA= +github.com/bufbuild/protoyaml-go v0.1.6 h1:wcdVCJOepw2Xd1KC53RuXoXkk4bc25gbFceFKN8QaO0= +github.com/bufbuild/protoyaml-go v0.1.6/go.mod h1:zrn7hI6KzmuW6kb8E437j3NAI2jY60eImQgxTInvcT8= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= diff --git a/private/buf/bufcli/bufcli.go b/private/buf/bufcli/bufcli.go index 645e1b8766..6ea87f242b 100644 --- a/private/buf/bufcli/bufcli.go +++ b/private/buf/bufcli/bufcli.go @@ -487,7 +487,7 @@ func NewWireImageReader( ) bufwire.ImageReader { return bufwire.NewImageReader( logger, - newFetchImageReader(logger, storageosProvider, runner), + newFetchMessageReader(logger, storageosProvider, runner), ) } @@ -506,9 +506,12 @@ func NewWireImageWriter( // NewWireProtoEncodingReader returns a new ProtoEncodingReader. func NewWireProtoEncodingReader( logger *zap.Logger, + storageosProvider storageos.Provider, + runner command.Runner, ) bufwire.ProtoEncodingReader { return bufwire.NewProtoEncodingReader( logger, + newFetchMessageReader(logger, storageosProvider, runner), ) } @@ -518,6 +521,9 @@ func NewWireProtoEncodingWriter( ) bufwire.ProtoEncodingWriter { return bufwire.NewProtoEncodingWriter( logger, + buffetch.NewWriter( + logger, + ), ) } @@ -1010,14 +1016,14 @@ func newFetchSourceReader( ) } -// newFetchImageReader creates a new buffetch.ImageReader with the default HTTP client +// newFetchMessageReader creates a new buffetch.MessageReader with the default HTTP client // and git cloner. -func newFetchImageReader( +func newFetchMessageReader( logger *zap.Logger, storageosProvider storageos.Provider, runner command.Runner, -) buffetch.ImageReader { - return buffetch.NewImageReader( +) buffetch.MessageReader { + return buffetch.NewMessageReader( logger, storageosProvider, defaultHTTPClient, diff --git a/private/buf/bufconvert/bufconvert.go b/private/buf/bufconvert/bufconvert.go deleted file mode 100644 index ac8ce77825..0000000000 --- a/private/buf/bufconvert/bufconvert.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufconvert - -import ( - "context" - "fmt" - "path/filepath" - "strings" - - "github.com/bufbuild/buf/private/buf/bufref" - "github.com/bufbuild/buf/private/pkg/app" - "github.com/bufbuild/buf/private/pkg/stringutil" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/codes" -) - -const ( - // MessageEncodingBinpb is the binary image encoding. - MessageEncodingBinpb MessageEncoding = iota + 1 - // MessageEncodingJSON is the JSON image encoding. - MessageEncodingJSON - // MessageEncodingTxtpb is the protobuf text image encoding. - MessageEncodingTxtpb - // formatBinpb is the binary format. - formatBinpb = "binpb" - // formatJSON is the JSON format. - formatJSON = "json" - // formatTxtpb is the protobuf text format. - formatTxtpb = "txtpb" - - // formatBin is the binary format's old form, now deprecated. - formatBin = "bin" -) - -var ( - // MessageEncodingFormatsString is the string representation of all message encoding formats. - // - // This does not include deprecated formats. - MessageEncodingFormatsString = stringutil.SliceToString(messageEncodingFormats) - // sorted - messageEncodingFormats = []string{ - formatBinpb, - formatJSON, - formatTxtpb, - } -) - -// MessageEncoding is the encoding of the message -type MessageEncoding int - -// MessageEncodingRef is a message encoding file reference. -type MessageEncodingRef interface { - Path() string - MessageEncoding() MessageEncoding -} - -// NewMessageEncodingRef returns a new MessageEncodingRef. -func NewMessageEncodingRef( - ctx context.Context, - value string, - defaultEncoding MessageEncoding, -) (MessageEncodingRef, error) { - ctx, span := otel.GetTracerProvider().Tracer("bufbuild/buf").Start(ctx, "new_message_encoding_ref") - defer span.End() - path, messageEncoding, err := getPathAndMessageEncoding(ctx, value, defaultEncoding) - if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - return nil, err - } - return newMessageEncodingRef(path, messageEncoding), nil -} - -func getPathAndMessageEncoding( - ctx context.Context, - value string, - defaultEncoding MessageEncoding, -) (string, MessageEncoding, error) { - path, options, err := bufref.GetRawPathAndOptions(value) - if err != nil { - return "", 0, err - } - messageEncoding := parseMessageEncodingExt(filepath.Ext(path), defaultEncoding) - for key, value := range options { - switch key { - case "format": - if app.IsDevNull(path) { - return "", 0, fmt.Errorf("not allowed if path is %s", app.DevNullFilePath) - } - messageEncoding, err = parseMessageEncodingFormat(value) - if err != nil { - return "", 0, err - } - default: - return "", 0, fmt.Errorf("invalid options key: %q", key) - } - } - return path, messageEncoding, nil -} - -func parseMessageEncodingExt(ext string, defaultEncoding MessageEncoding) MessageEncoding { - switch strings.TrimPrefix(ext, ".") { - case formatBin, formatBinpb: - return MessageEncodingBinpb - case formatJSON: - return MessageEncodingJSON - case formatTxtpb: - return MessageEncodingTxtpb - default: - return defaultEncoding - } -} - -func parseMessageEncodingFormat(format string) (MessageEncoding, error) { - switch format { - case formatBin, formatBinpb: - return MessageEncodingBinpb, nil - case formatJSON: - return MessageEncodingJSON, nil - case formatTxtpb: - return MessageEncodingTxtpb, nil - default: - return 0, fmt.Errorf("invalid format for message: %q", format) - } -} diff --git a/private/buf/bufconvert/message_encoding_ref.go b/private/buf/bufconvert/message_encoding_ref.go deleted file mode 100644 index 3790ba75e3..0000000000 --- a/private/buf/bufconvert/message_encoding_ref.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufconvert - -var _ MessageEncodingRef = &messageEncodingRef{} - -type messageEncodingRef struct { - path string - messageEncoding MessageEncoding -} - -func newMessageEncodingRef( - path string, - messageEncoding MessageEncoding, -) *messageEncodingRef { - return &messageEncodingRef{ - path: path, - messageEncoding: messageEncoding, - } -} - -func (r *messageEncodingRef) Path() string { - return r.path -} - -func (r *messageEncodingRef) MessageEncoding() MessageEncoding { - return r.messageEncoding -} diff --git a/private/buf/bufconvert/usage.gen.go b/private/buf/bufconvert/usage.gen.go deleted file mode 100644 index 29527dd3b1..0000000000 --- a/private/buf/bufconvert/usage.gen.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Generated. DO NOT EDIT. - -package bufconvert - -import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/buffetch/buffetch.go b/private/buf/buffetch/buffetch.go index 91b7d94c5c..e30a939852 100644 --- a/private/buf/buffetch/buffetch.go +++ b/private/buf/buffetch/buffetch.go @@ -30,19 +30,24 @@ import ( ) const ( - // ImageEncodingBin is the binary image encoding. - ImageEncodingBin ImageEncoding = iota + 1 - // ImageEncodingJSON is the JSON image encoding. - ImageEncodingJSON - // ImageEncodingTxtpb is the text protobuf image encoding. - ImageEncodingTxtpb + // MessageEncodingBinpb is the binary message encoding. + MessageEncodingBinpb MessageEncoding = iota + 1 + // MessageEncodingJSON is the JSON message encoding. + MessageEncodingJSON + // MessageEncodingTxtpb is the text protobuf message encoding. + MessageEncodingTxtpb + // MessageEncodingYAML is the YAML message encoding. + MessageEncodingYAML + + useProtoNamesKey = "use_proto_names" + useEnumNumbersKey = "use_enum_numbers" ) var ( - // ImageFormatsString is the string representation of all image formats. + // MessageFormatsString is the string representation of all message formats. // // This does not include deprecated formats. - ImageFormatsString = stringutil.SliceToString(imageFormatsNotDeprecated) + MessageFormatsString = stringutil.SliceToString(messageFormatsNotDeprecated) // SourceDirFormatsString is the string representation of all source directory formats. // This includes all of the formats in SourceFormatsString except the protofile format. // @@ -68,8 +73,8 @@ var ( AllFormatsString = stringutil.SliceToString(allFormatsNotDeprecated) ) -// ImageEncoding is the encoding of the image. -type ImageEncoding int +// MessageEncoding is the encoding of the message. +type MessageEncoding int // PathResolver resolves external paths to paths. type PathResolver interface { @@ -90,19 +95,27 @@ type PathResolver interface { PathForExternalPath(externalPath string) (string, error) } -// Ref is an image file or source bucket reference. +// Ref is an message file or source bucket reference. type Ref interface { PathResolver internalRef() internal.Ref } -// ImageRef is an image file reference. -type ImageRef interface { +// MessageRef is an message file reference. +type MessageRef interface { Ref - ImageEncoding() ImageEncoding + MessageEncoding() MessageEncoding + // Path returns the path of the file. + // + // May be used for items such as YAML unmarshaling errors. + Path() string + // UseProtoNames only applies for MessageEncodingYAML at this time. + UseProtoNames() bool + // UseEnumNumbers only applies for MessageEncodingYAML at this time. + UseEnumNumbers() bool IsNull() bool - internalFileRef() internal.FileRef + internalSingleRef() internal.SingleRef } // SourceOrModuleRef is a source bucket or module reference. @@ -130,10 +143,10 @@ type ProtoFileRef interface { internalProtoFileRef() internal.ProtoFileRef } -// ImageRefParser is an image ref parser for Buf. -type ImageRefParser interface { - // GetImageRef gets the reference for the image file. - GetImageRef(ctx context.Context, value string) (ImageRef, error) +// MessageRefParser is an message ref parser for Buf. +type MessageRefParser interface { + // GetMessageRef gets the reference for the message file. + GetMessageRef(ctx context.Context, value string) (MessageRef, error) } // SourceRefParser is a source ref parser for Buf. @@ -155,16 +168,16 @@ type SourceOrModuleRefParser interface { SourceRefParser ModuleRefParser - // GetSourceOrModuleRef gets the reference for the image file or source bucket. + // GetSourceOrModuleRef gets the reference for the message file or source bucket. GetSourceOrModuleRef(ctx context.Context, value string) (SourceOrModuleRef, error) } // RefParser is a ref parser for Buf. type RefParser interface { - ImageRefParser + MessageRefParser SourceOrModuleRefParser - // GetRef gets the reference for the image file, source bucket, or module. + // GetRef gets the reference for the message file, source bucket, or module. GetRef(ctx context.Context, value string) (Ref, error) } @@ -175,11 +188,21 @@ func NewRefParser(logger *zap.Logger) RefParser { return newRefParser(logger) } -// NewImageRefParser returns a new RefParser for images only. +// NewMessageRefParser returns a new RefParser for messages only. +func NewMessageRefParser(logger *zap.Logger, options ...MessageRefParserOption) MessageRefParser { + return newMessageRefParser(logger, options...) +} + +// MessageRefParserOption is an option for a new MessageRefParser. +type MessageRefParserOption func(*messageRefParserOptions) + +// MessageRefParserWithDefaultMessageEncoding says to use the default MessageEncoding. // -// This defaults to binary. -func NewImageRefParser(logger *zap.Logger) ImageRefParser { - return newImageRefParser(logger) +// The default default is MessageEncodingBinpb. +func MessageRefParserWithDefaultMessageEncoding(defaultMessageEncoding MessageEncoding) MessageRefParserOption { + return func(messageRefParserOptions *messageRefParserOptions) { + messageRefParserOptions.defaultMessageEncoding = defaultMessageEncoding + } } // NewSourceRefParser returns a new RefParser for sources only. @@ -216,15 +239,15 @@ type ReadWriteBucketCloser internal.ReadWriteBucketCloser // ReadBucketCloserWithTerminateFileProvider is a ReadBucketCloser with a TerminateFileProvider. type ReadBucketCloserWithTerminateFileProvider internal.ReadBucketCloserWithTerminateFileProvider -// ImageReader is an image reader. -type ImageReader interface { - // GetImageFile gets the image file. +// MessageReader is an message reader. +type MessageReader interface { + // GetMessageFile gets the message file. // // The returned file will be uncompressed. - GetImageFile( + GetMessageFile( ctx context.Context, container app.EnvStdinContainer, - imageRef ImageRef, + messageRef MessageRef, ) (io.ReadCloser, error) } @@ -265,7 +288,7 @@ type ModuleFetcher interface { // Reader is a reader for Buf. type Reader interface { - ImageReader + MessageReader SourceReader ModuleFetcher } @@ -291,15 +314,15 @@ func NewReader( ) } -// NewImageReader returns a new ImageReader. -func NewImageReader( +// NewMessageReader returns a new MessageReader. +func NewMessageReader( logger *zap.Logger, storageosProvider storageos.Provider, httpClient *http.Client, httpAuthenticator httpauth.Authenticator, gitCloner git.Cloner, -) ImageReader { - return newImageReader( +) MessageReader { + return newMessageReader( logger, storageosProvider, httpClient, @@ -342,11 +365,11 @@ func NewModuleFetcher( // Writer is a writer for Buf. type Writer interface { - // PutImageFile puts the image file. - PutImageFile( + // PutMessageFile puts the message file. + PutMessageFile( ctx context.Context, container app.EnvStdoutContainer, - imageRef ImageRef, + messageRef MessageRef, ) (io.WriteCloser, error) } diff --git a/private/buf/buffetch/format.go b/private/buf/buffetch/format.go index 8a3079b8ed..7ea7bcd0e9 100644 --- a/private/buf/buffetch/format.go +++ b/private/buf/buffetch/format.go @@ -25,6 +25,8 @@ const ( formatGit = "git" // formatJSON is the JSON format. formatJSON = "json" + // formatYAML is the YAML format. + formatYAML = "yaml" // formatMod is the module format. formatMod = "mod" // formatTar is the tar format. @@ -46,19 +48,21 @@ const ( var ( // sorted - imageFormats = []string{ + messageFormats = []string{ formatBin, formatBinpb, formatBingz, formatJSON, formatJSONGZ, formatTxtpb, + formatYAML, } // sorted - imageFormatsNotDeprecated = []string{ + messageFormatsNotDeprecated = []string{ formatBinpb, formatJSON, formatTxtpb, + formatYAML, } // sorted sourceFormats = []string{ @@ -124,6 +128,7 @@ var ( formatTar, formatTargz, formatTxtpb, + formatYAML, formatZip, } // sorted @@ -136,6 +141,7 @@ var ( formatProtoFile, formatTar, formatTxtpb, + formatYAML, formatZip, } @@ -144,4 +150,11 @@ var ( formatJSONGZ: formatJSON, formatTargz: formatTar, } + + messageEncodingToFormat = map[MessageEncoding]string{ + MessageEncodingBinpb: formatBinpb, + MessageEncodingJSON: formatJSON, + MessageEncodingTxtpb: formatTxtpb, + MessageEncodingYAML: formatYAML, + } ) diff --git a/private/buf/buffetch/image_ref.go b/private/buf/buffetch/image_ref.go deleted file mode 100644 index b9eb995228..0000000000 --- a/private/buf/buffetch/image_ref.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package buffetch - -import ( - "github.com/bufbuild/buf/private/buf/buffetch/internal" - "github.com/bufbuild/buf/private/pkg/normalpath" -) - -var _ ImageRef = &imageRef{} - -type imageRef struct { - fileRef internal.FileRef - imageEncoding ImageEncoding -} - -func newImageRef( - fileRef internal.FileRef, - imageEncoding ImageEncoding, -) *imageRef { - return &imageRef{ - fileRef: fileRef, - imageEncoding: imageEncoding, - } -} - -func (r *imageRef) PathForExternalPath(externalPath string) (string, error) { - return normalpath.NormalizeAndValidate(externalPath) -} - -func (r *imageRef) ImageEncoding() ImageEncoding { - return r.imageEncoding -} - -func (r *imageRef) IsNull() bool { - return r.fileRef.FileScheme() == internal.FileSchemeNull -} - -func (r *imageRef) internalRef() internal.Ref { - return r.fileRef -} - -func (r *imageRef) internalFileRef() internal.FileRef { - return r.fileRef -} diff --git a/private/buf/buffetch/internal/archive_ref.go b/private/buf/buffetch/internal/archive_ref.go index 09552f0289..9ca53be8ad 100644 --- a/private/buf/buffetch/internal/archive_ref.go +++ b/private/buf/buffetch/internal/archive_ref.go @@ -45,6 +45,7 @@ func newArchiveRef( format, path, compressionType, + nil, ) if err != nil { return nil, err diff --git a/private/buf/buffetch/internal/errors.go b/private/buf/buffetch/internal/errors.go index 1c9cd4ac13..bcc11d8da7 100644 --- a/private/buf/buffetch/internal/errors.go +++ b/private/buf/buffetch/internal/errors.go @@ -78,9 +78,17 @@ func NewNoPathError() error { return errors.New("value has no path once processed") } -// NewOptionsInvalidKeyError is a fetch error. -func NewOptionsInvalidKeyError(key string) error { - return fmt.Errorf("invalid options key: %q", key) +// NewOptionsInvalidKeysError is a fetch error. +func NewOptionsInvalidKeysError(keys ...string) error { + if len(keys) == 1 { + return fmt.Errorf("invalid key: %q", keys[0]) + } + return fmt.Errorf("invalid keys: \"%v\"", strings.Join(keys, ", ")) +} + +// NewOptionsInvalidValueForKeyError is a fetch error. +func NewOptionsInvalidValueForKeyError(key string, value string) error { + return fmt.Errorf("invalid value %q for key: %q", value, key) } // NewOptionsInvalidForFormatError is a fetch error. @@ -178,3 +186,27 @@ func NewWriteLocalDisabledError() error { func NewWriteStdioDisabledError() error { return NewWriteDisabledError("stdout") } + +func newValueEmptyError() error { + return errors.New("required") +} + +func newValueMultipleHashtagsError(value string) error { + return fmt.Errorf("%q has multiple #s which is invalid", value) +} + +func newValueStartsWithHashtagError(value string) error { + return fmt.Errorf("%q starts with # which is invalid", value) +} + +func newValueEndsWithHashtagError(value string) error { + return fmt.Errorf("%q ends with # which is invalid", value) +} + +func newOptionsInvalidError(s string) error { + return fmt.Errorf("invalid options: %q", s) +} + +func newOptionsDuplicateKeyError(key string) error { + return fmt.Errorf("duplicate options key: %q", key) +} diff --git a/private/buf/buffetch/internal/internal.go b/private/buf/buffetch/internal/internal.go index 5983dc7658..6e43764efd 100644 --- a/private/buf/buffetch/internal/internal.go +++ b/private/buf/buffetch/internal/internal.go @@ -112,12 +112,13 @@ type BucketRef interface { // SingleRef is a non-archive file reference. type SingleRef interface { FileRef + CustomOptionValue(key string) (string, bool) singleRef() } // NewSingleRef returns a new SingleRef. func NewSingleRef(path string, compressionType CompressionType) (SingleRef, error) { - return newSingleRef("", path, compressionType) + return newSingleRef("", path, compressionType, nil) } // ArchiveRef is an archive reference. @@ -252,12 +253,14 @@ func NewDirectParsedSingleRef( path string, fileScheme FileScheme, compressionType CompressionType, + customOptions map[string]string, ) ParsedSingleRef { return newDirectSingleRef( format, path, fileScheme, compressionType, + customOptions, ) } @@ -528,9 +531,13 @@ type RawRef struct { ArchiveStripComponents uint32 // Only set for proto file ref format. // Sets whether or not to include the files in the rest of the package - // in the image for the ProtoFileRef. + // in the message for the ProtoFileRef. // This defaults to false. IncludePackageFiles bool + // Any unrecognized options. Some formats may allow custom options, and those + // formats should check for those custom options in this map. If a format + // does not allow an option, an error will be returned. + UnrecognizedOptions map[string]string } // RefParserOption is an RefParser option. @@ -658,6 +665,13 @@ func WithSingleDefaultCompressionType(defaultCompressionType CompressionType) Si } } +// WithSingleCustomOptionKey adds a custom option key that is recognized.. +func WithSingleCustomOptionKey(key string) SingleFormatOption { + return func(singleFormatInfo *singleFormatInfo) { + singleFormatInfo.customOptionKeys[key] = struct{}{} + } +} + // ArchiveFormatOption is a archive format option. type ArchiveFormatOption func(*archiveFormatInfo) diff --git a/private/buf/buffetch/internal/ref_parser.go b/private/buf/buffetch/internal/ref_parser.go index 49071108c4..e88c61a15a 100644 --- a/private/buf/buffetch/internal/ref_parser.go +++ b/private/buf/buffetch/internal/ref_parser.go @@ -16,9 +16,10 @@ package internal import ( "context" + "sort" "strconv" + "strings" - "github.com/bufbuild/buf/private/buf/bufref" "github.com/bufbuild/buf/private/pkg/app" "github.com/bufbuild/buf/private/pkg/git" "github.com/bufbuild/buf/private/pkg/normalpath" @@ -87,8 +88,19 @@ func (a *refParser) getParsedRef( return nil, NewFormatNotAllowedError(rawRef.Format, allowedFormats) } } + if !singleOK && len(rawRef.UnrecognizedOptions) > 0 { + // Only single refs allow custom options. In every other case, this is an error. + // + // We verify unrecognized options match what is expected in getSingleRef. + keys := make([]string, 0, len(rawRef.UnrecognizedOptions)) + for key := range rawRef.UnrecognizedOptions { + keys = append(keys, key) + } + sort.Strings(keys) + return nil, NewOptionsInvalidKeysError(keys...) + } if singleOK { - return getSingleRef(rawRef, singleFormatInfo.defaultCompressionType) + return getSingleRef(rawRef, singleFormatInfo.defaultCompressionType, singleFormatInfo.customOptionKeys) } if archiveOK { return getArchiveRef(rawRef, archiveFormatInfo.archiveType, archiveFormatInfo.defaultCompressionType) @@ -111,12 +123,13 @@ func (a *refParser) getParsedRef( // validated per rules on rawRef func (a *refParser) getRawRef(value string) (*RawRef, error) { // path is never empty after returning from this function - path, options, err := bufref.GetRawPathAndOptions(value) + path, options, err := getRawPathAndOptions(value) if err != nil { return nil, err } rawRef := &RawRef{ - Path: path, + Path: path, + UnrecognizedOptions: make(map[string]string), } if a.rawRefProcessor != nil { if err := a.rawRefProcessor(rawRef); err != nil { @@ -195,10 +208,10 @@ func (a *refParser) getRawRef(value string) (*RawRef, error) { case "false", "": rawRef.IncludePackageFiles = false default: - return nil, NewOptionsInvalidKeyError(key) + return nil, NewOptionsInvalidValueForKeyError(key, value) } default: - return nil, NewOptionsInvalidKeyError(key) + rawRef.UnrecognizedOptions[key] = value } } @@ -249,18 +262,71 @@ func (a *refParser) getRawRef(value string) (*RawRef, error) { return rawRef, nil } +// getRawPathAndOptions returns the raw path and options from the value provided, +// the rawPath will be non-empty when returning without error here. +func getRawPathAndOptions(value string) (string, map[string]string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "", nil, newValueEmptyError() + } + + switch splitValue := strings.Split(value, "#"); len(splitValue) { + case 1: + return value, nil, nil + case 2: + path := strings.TrimSpace(splitValue[0]) + optionsString := strings.TrimSpace(splitValue[1]) + if path == "" { + return "", nil, newValueStartsWithHashtagError(value) + } + if optionsString == "" { + return "", nil, newValueEndsWithHashtagError(value) + } + options := make(map[string]string) + for _, pair := range strings.Split(optionsString, ",") { + split := strings.Split(pair, "=") + if len(split) != 2 { + return "", nil, newOptionsInvalidError(optionsString) + } + key := strings.TrimSpace(split[0]) + value := strings.TrimSpace(split[1]) + if key == "" || value == "" { + return "", nil, newOptionsInvalidError(optionsString) + } + if _, ok := options[key]; ok { + return "", nil, newOptionsDuplicateKeyError(key) + } + options[key] = value + } + return path, options, nil + default: + return "", nil, newValueMultipleHashtagsError(value) + } +} + func getSingleRef( rawRef *RawRef, defaultCompressionType CompressionType, + customOptionKeys map[string]struct{}, ) (ParsedSingleRef, error) { compressionType := rawRef.CompressionType if compressionType == 0 { compressionType = defaultCompressionType } + var invalidKeys []string + for key := range rawRef.UnrecognizedOptions { + if _, ok := customOptionKeys[key]; !ok { + invalidKeys = append(invalidKeys, key) + } + } + if len(invalidKeys) > 0 { + return nil, NewOptionsInvalidKeysError(invalidKeys...) + } return newSingleRef( rawRef.Format, rawRef.Path, compressionType, + rawRef.UnrecognizedOptions, ) } @@ -354,11 +420,13 @@ func getProtoFileRef(rawRef *RawRef) (ParsedProtoFileRef, error) { type singleFormatInfo struct { defaultCompressionType CompressionType + customOptionKeys map[string]struct{} } func newSingleFormatInfo() *singleFormatInfo { return &singleFormatInfo{ defaultCompressionType: CompressionTypeNone, + customOptionKeys: make(map[string]struct{}), } } diff --git a/private/buf/bufref/bufref_test.go b/private/buf/buffetch/internal/ref_parser_test.go similarity index 97% rename from private/buf/bufref/bufref_test.go rename to private/buf/buffetch/internal/ref_parser_test.go index 9fe16ddfbb..83fbd6a721 100644 --- a/private/buf/bufref/bufref_test.go +++ b/private/buf/buffetch/internal/ref_parser_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package bufref +package internal import ( "testing" @@ -81,7 +81,7 @@ func testGetRawPathAndOptionsError( ) { t.Run(value, func(t *testing.T) { t.Parallel() - _, _, err := GetRawPathAndOptions(value) + _, _, err := getRawPathAndOptions(value) assert.EqualError(t, err, expectedErr.Error()) }) } diff --git a/private/buf/buffetch/internal/single_ref.go b/private/buf/buffetch/internal/single_ref.go index addd35333c..1fc1ddebd1 100644 --- a/private/buf/buffetch/internal/single_ref.go +++ b/private/buf/buffetch/internal/single_ref.go @@ -36,12 +36,14 @@ type singleRef struct { path string fileScheme FileScheme compressionType CompressionType + customOptions map[string]string } func newSingleRef( format string, path string, compressionType CompressionType, + customOptions map[string]string, ) (*singleRef, error) { if path == "" { return nil, NewNoPathError() @@ -55,6 +57,7 @@ func newSingleRef( "", FileSchemeStdio, compressionType, + customOptions, ), nil } if app.IsDevStdin(path) { @@ -63,6 +66,7 @@ func newSingleRef( "", FileSchemeStdin, compressionType, + customOptions, ), nil } if app.IsDevStdout(path) { @@ -71,6 +75,7 @@ func newSingleRef( "", FileSchemeStdout, compressionType, + customOptions, ), nil } if app.IsDevNull(path) { @@ -79,6 +84,7 @@ func newSingleRef( "", FileSchemeNull, compressionType, + customOptions, ), nil } for prefix, fileScheme := range fileSchemePrefixToFileScheme { @@ -95,6 +101,7 @@ func newSingleRef( path, fileScheme, compressionType, + customOptions, ), nil } } @@ -106,6 +113,7 @@ func newSingleRef( normalpath.Normalize(path), FileSchemeLocal, compressionType, + customOptions, ), nil } @@ -114,12 +122,17 @@ func newDirectSingleRef( path string, fileScheme FileScheme, compressionType CompressionType, + customOptions map[string]string, ) *singleRef { + if customOptions == nil { + customOptions = make(map[string]string) + } return &singleRef{ format: format, path: path, fileScheme: fileScheme, compressionType: compressionType, + customOptions: customOptions, } } @@ -139,6 +152,11 @@ func (r *singleRef) CompressionType() CompressionType { return r.compressionType } +func (r *singleRef) CustomOptionValue(key string) (string, bool) { + value, ok := r.customOptions[key] + return value, ok +} + func (*singleRef) ref() {} func (*singleRef) fileRef() {} func (*singleRef) singleRef() {} diff --git a/private/buf/buffetch/message_ref.go b/private/buf/buffetch/message_ref.go new file mode 100644 index 0000000000..ce3a2d16d2 --- /dev/null +++ b/private/buf/buffetch/message_ref.go @@ -0,0 +1,96 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffetch + +import ( + "github.com/bufbuild/buf/private/buf/buffetch/internal" + "github.com/bufbuild/buf/private/pkg/normalpath" +) + +var _ MessageRef = &messageRef{} + +type messageRef struct { + singleRef internal.SingleRef + useProtoNames bool + useEnumNumbers bool + messageEncoding MessageEncoding +} + +func newMessageRef( + singleRef internal.SingleRef, + messageEncoding MessageEncoding, +) (*messageRef, error) { + useProtoNames, err := getTrueOrFalseForSingleRef(singleRef, useProtoNamesKey) + if err != nil { + return nil, err + } + useEnumNumbers, err := getTrueOrFalseForSingleRef(singleRef, useEnumNumbersKey) + if err != nil { + return nil, err + } + return &messageRef{ + singleRef: singleRef, + useProtoNames: useProtoNames, + useEnumNumbers: useEnumNumbers, + messageEncoding: messageEncoding, + }, nil +} + +func (r *messageRef) PathForExternalPath(externalPath string) (string, error) { + return normalpath.NormalizeAndValidate(externalPath) +} + +func (r *messageRef) MessageEncoding() MessageEncoding { + return r.messageEncoding +} + +func (r *messageRef) Path() string { + return r.singleRef.Path() +} + +func (r *messageRef) UseProtoNames() bool { + return r.useProtoNames +} + +func (r *messageRef) UseEnumNumbers() bool { + return r.useEnumNumbers +} + +func (r *messageRef) IsNull() bool { + return r.singleRef.FileScheme() == internal.FileSchemeNull +} + +func (r *messageRef) internalRef() internal.Ref { + return r.singleRef +} + +func (r *messageRef) internalSingleRef() internal.SingleRef { + return r.singleRef +} + +func getTrueOrFalseForSingleRef(singleRef internal.SingleRef, key string) (bool, error) { + value, ok := singleRef.CustomOptionValue(key) + if !ok { + return false, nil + } + switch value { + case "true": + return true, nil + case "false": + return false, nil + default: + return false, internal.NewOptionsInvalidValueForKeyError(key, value) + } +} diff --git a/private/buf/buffetch/reader.go b/private/buf/buffetch/reader.go index 8194da79c2..a18b7f5b9c 100644 --- a/private/buf/buffetch/reader.go +++ b/private/buf/buffetch/reader.go @@ -64,7 +64,7 @@ func newReader( } } -func newImageReader( +func newMessageReader( logger *zap.Logger, storageosProvider storageos.Provider, httpClient *http.Client, @@ -127,12 +127,12 @@ func newModuleFetcher( } } -func (a *reader) GetImageFile( +func (a *reader) GetMessageFile( ctx context.Context, container app.EnvStdinContainer, - imageRef ImageRef, + messageRef MessageRef, ) (io.ReadCloser, error) { - return a.internalReader.GetFile(ctx, container, imageRef.internalFileRef()) + return a.internalReader.GetFile(ctx, container, messageRef.internalSingleRef()) } func (a *reader) GetSourceBucket( diff --git a/private/buf/buffetch/ref_parser.go b/private/buf/buffetch/ref_parser.go index bddc380333..a252fca81f 100644 --- a/private/buf/buffetch/ref_parser.go +++ b/private/buf/buffetch/ref_parser.go @@ -48,11 +48,20 @@ func newRefParser(logger *zap.Logger) *refParser { tracer: otel.GetTracerProvider().Tracer(tracerName), fetchRefParser: internal.NewRefParser( logger, - internal.WithRawRefProcessor(newRawRefProcessor()), + internal.WithRawRefProcessor(processRawRef), internal.WithSingleFormat(formatBin), internal.WithSingleFormat(formatBinpb), - internal.WithSingleFormat(formatJSON), + internal.WithSingleFormat( + formatJSON, + internal.WithSingleCustomOptionKey(useProtoNamesKey), + internal.WithSingleCustomOptionKey(useEnumNumbersKey), + ), internal.WithSingleFormat(formatTxtpb), + internal.WithSingleFormat( + formatYAML, + internal.WithSingleCustomOptionKey(useProtoNamesKey), + internal.WithSingleCustomOptionKey(useEnumNumbersKey), + ), internal.WithSingleFormat( formatBingz, internal.WithSingleDefaultCompressionType( @@ -88,16 +97,29 @@ func newRefParser(logger *zap.Logger) *refParser { } } -func newImageRefParser(logger *zap.Logger) *refParser { +func newMessageRefParser(logger *zap.Logger, options ...MessageRefParserOption) *refParser { + messageRefParserOptions := newMessageRefParserOptions() + for _, option := range options { + option(messageRefParserOptions) + } return &refParser{ logger: logger.Named(loggerName), fetchRefParser: internal.NewRefParser( logger, - internal.WithRawRefProcessor(processRawRefImage), + internal.WithRawRefProcessor(newProcessRawRefMessage(messageRefParserOptions.defaultMessageEncoding)), internal.WithSingleFormat(formatBin), internal.WithSingleFormat(formatBinpb), - internal.WithSingleFormat(formatJSON), + internal.WithSingleFormat( + formatJSON, + internal.WithSingleCustomOptionKey(useProtoNamesKey), + internal.WithSingleCustomOptionKey(useEnumNumbersKey), + ), internal.WithSingleFormat(formatTxtpb), + internal.WithSingleFormat( + formatYAML, + internal.WithSingleCustomOptionKey(useProtoNamesKey), + internal.WithSingleCustomOptionKey(useEnumNumbersKey), + ), internal.WithSingleFormat( formatBingz, internal.WithSingleDefaultCompressionType( @@ -202,11 +224,11 @@ func (a *refParser) GetRef( } switch t := parsedRef.(type) { case internal.ParsedSingleRef: - imageEncoding, err := parseImageEncoding(t.Format()) + messageEncoding, err := parseMessageEncoding(t.Format()) if err != nil { return nil, err } - return newImageRef(t, imageEncoding), nil + return newMessageRef(t, messageEncoding) case internal.ParsedArchiveRef: return newSourceRef(t), nil case internal.ParsedDirRef: @@ -256,11 +278,11 @@ func (a *refParser) GetSourceOrModuleRef( } } -func (a *refParser) GetImageRef( +func (a *refParser) GetMessageRef( ctx context.Context, value string, -) (_ ImageRef, retErr error) { - ctx, span := a.tracer.Start(ctx, "get_image_ref") +) (_ MessageRef, retErr error) { + ctx, span := a.tracer.Start(ctx, "get_message_ref") defer span.End() defer func() { if retErr != nil { @@ -268,19 +290,19 @@ func (a *refParser) GetImageRef( span.SetStatus(codes.Error, retErr.Error()) } }() - parsedRef, err := a.getParsedRef(ctx, value, imageFormats) + parsedRef, err := a.getParsedRef(ctx, value, messageFormats) if err != nil { return nil, err } parsedSingleRef, ok := parsedRef.(internal.ParsedSingleRef) if !ok { - return nil, fmt.Errorf("invalid ParsedRef type for image: %T", parsedRef) + return nil, fmt.Errorf("invalid ParsedRef type for message: %T", parsedRef) } - imageEncoding, err := parseImageEncoding(parsedSingleRef.Format()) + messageEncoding, err := parseMessageEncoding(parsedSingleRef.Format()) if err != nil { return nil, err } - return newImageRef(parsedSingleRef, imageEncoding), nil + return newMessageRef(parsedSingleRef, messageEncoding) } func (a *refParser) GetSourceRef( @@ -359,15 +381,29 @@ func (a *refParser) checkDeprecated(parsedRef internal.ParsedRef) { } } -func newRawRefProcessor() func(*internal.RawRef) error { - return func(rawRef *internal.RawRef) error { - // if format option is not set and path is "-", default to bin - var format string - var compressionType internal.CompressionType - if rawRef.Path == "-" || app.IsDevNull(rawRef.Path) || app.IsDevStdin(rawRef.Path) || app.IsDevStdout(rawRef.Path) { +func processRawRef(rawRef *internal.RawRef) error { + // if format option is not set and path is "-", default to bin + var format string + var compressionType internal.CompressionType + if rawRef.Path == "-" || app.IsDevNull(rawRef.Path) || app.IsDevStdin(rawRef.Path) || app.IsDevStdout(rawRef.Path) { + format = formatBinpb + } else { + switch filepath.Ext(rawRef.Path) { + case ".bin", ".binpb": format = formatBinpb - } else { - switch filepath.Ext(rawRef.Path) { + case ".json": + format = formatJSON + case ".tar": + format = formatTar + case ".txtpb": + format = formatTxtpb + case ".yaml": + format = formatYAML + case ".zip": + format = formatZip + case ".gz": + compressionType = internal.CompressionTypeGzip + switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { case ".bin", ".binpb": format = formatBinpb case ".json": @@ -376,64 +412,54 @@ func newRawRefProcessor() func(*internal.RawRef) error { format = formatTar case ".txtpb": format = formatTxtpb - case ".zip": - format = formatZip - case ".gz": - compressionType = internal.CompressionTypeGzip - switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { - case ".bin", ".binpb": - format = formatBinpb - case ".json": - format = formatJSON - case ".tar": - format = formatTar - case ".txtpb": - format = formatTxtpb - default: - return fmt.Errorf("path %q had .gz extension with unknown format", rawRef.Path) - } - case ".zst": - compressionType = internal.CompressionTypeZstd - switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { - case ".bin", ".binpb": - format = formatBinpb - case ".json": - format = formatJSON - case ".tar": - format = formatTar - case ".txtpb": - format = formatTxtpb - default: - return fmt.Errorf("path %q had .zst extension with unknown format", rawRef.Path) - } - case ".tgz": + case ".yaml": + format = formatYAML + default: + return fmt.Errorf("path %q had .gz extension with unknown format", rawRef.Path) + } + case ".zst": + compressionType = internal.CompressionTypeZstd + switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { + case ".bin", ".binpb": + format = formatBinpb + case ".json": + format = formatJSON + case ".tar": format = formatTar - compressionType = internal.CompressionTypeGzip - case ".git": - format = formatGit - // This only applies if the option accept `ProtoFileRef` is passed in, otherwise - // it falls through to the `default` case. - case ".proto": - fileInfo, err := os.Stat(rawRef.Path) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("path provided is not a valid proto file: %s, %w", rawRef.Path, err) - } - if fileInfo != nil && fileInfo.IsDir() { - return fmt.Errorf("path provided is not a valid proto file: a directory named %s already exists", rawRef.Path) - } - format = formatProtoFile + case ".txtpb": + format = formatTxtpb + case ".yaml": + format = formatYAML default: - var err error - format, err = assumeModuleOrDir(rawRef.Path) - if err != nil { - return err - } + return fmt.Errorf("path %q had .zst extension with unknown format", rawRef.Path) + } + case ".tgz": + format = formatTar + compressionType = internal.CompressionTypeGzip + case ".git": + format = formatGit + // This only applies if the option accept `ProtoFileRef` is passed in, otherwise + // it falls through to the `default` case. + case ".proto": + fileInfo, err := os.Stat(rawRef.Path) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("path provided is not a valid proto file: %s, %w", rawRef.Path, err) + } + if fileInfo != nil && fileInfo.IsDir() { + return fmt.Errorf("path provided is not a valid proto file: a directory named %s already exists", rawRef.Path) + } + format = formatProtoFile + default: + var err error + format, err = assumeModuleOrDir(rawRef.Path) + if err != nil { + return err } } - rawRef.Format = format - rawRef.CompressionType = compressionType - return nil } + rawRef.Format = format + rawRef.CompressionType = compressionType + return nil } func processRawRefSource(rawRef *internal.RawRef) error { @@ -516,51 +542,64 @@ func processRawRefSourceOrModule(rawRef *internal.RawRef) error { return nil } -func processRawRefImage(rawRef *internal.RawRef) error { - // if format option is not set and path is "-", default to bin - var format string - var compressionType internal.CompressionType - if rawRef.Path == "-" || app.IsDevNull(rawRef.Path) || app.IsDevStdin(rawRef.Path) || app.IsDevStdout(rawRef.Path) { - format = formatBinpb - } else { - switch filepath.Ext(rawRef.Path) { - case ".bin", ".binpb": - format = formatBinpb - case ".json": - format = formatJSON - case ".txtpb": - format = formatTxtpb - case ".gz": - compressionType = internal.CompressionTypeGzip - switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { - case ".bin", ".binpb": - format = formatBinpb - case ".json": - format = formatJSON - case ".txtpb": - format = formatTxtpb - default: - return fmt.Errorf("path %q had .gz extension with unknown format", rawRef.Path) - } - case ".zst": - compressionType = internal.CompressionTypeZstd - switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { +func newProcessRawRefMessage(defaultMessageEncoding MessageEncoding) func(*internal.RawRef) error { + return func(rawRef *internal.RawRef) error { + defaultFormat, ok := messageEncodingToFormat[defaultMessageEncoding] + if !ok { + // This is a system error. + return fmt.Errorf("unknown MessageEncoding: %v", defaultMessageEncoding) + } + // if format option is not set and path is "-", default to bin + var format string + var compressionType internal.CompressionType + if rawRef.Path == "-" || app.IsDevNull(rawRef.Path) || app.IsDevStdin(rawRef.Path) || app.IsDevStdout(rawRef.Path) { + format = defaultFormat + } else { + switch filepath.Ext(rawRef.Path) { case ".bin", ".binpb": format = formatBinpb case ".json": format = formatJSON case ".txtpb": format = formatTxtpb + case ".yaml": + format = formatYAML + case ".gz": + compressionType = internal.CompressionTypeGzip + switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { + case ".bin", ".binpb": + format = formatBinpb + case ".json": + format = formatJSON + case ".txtpb": + format = formatTxtpb + case ".yaml": + format = formatYAML + default: + return fmt.Errorf("path %q had .gz extension with unknown format", rawRef.Path) + } + case ".zst": + compressionType = internal.CompressionTypeZstd + switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { + case ".bin", ".binpb": + format = formatBinpb + case ".json": + format = formatJSON + case ".txtpb": + format = formatTxtpb + case ".yaml": + format = formatYAML + default: + return fmt.Errorf("path %q had .zst extension with unknown format", rawRef.Path) + } default: - return fmt.Errorf("path %q had .zst extension with unknown format", rawRef.Path) + format = defaultFormat } - default: - format = formatBinpb } + rawRef.Format = format + rawRef.CompressionType = compressionType + return nil } - rawRef.Format = format - rawRef.CompressionType = compressionType - return nil } func processRawRefModule(rawRef *internal.RawRef) error { @@ -568,16 +607,18 @@ func processRawRefModule(rawRef *internal.RawRef) error { return nil } -func parseImageEncoding(format string) (ImageEncoding, error) { +func parseMessageEncoding(format string) (MessageEncoding, error) { switch format { case formatBin, formatBinpb, formatBingz: - return ImageEncodingBin, nil + return MessageEncodingBinpb, nil case formatJSON, formatJSONGZ: - return ImageEncodingJSON, nil + return MessageEncodingJSON, nil case formatTxtpb: - return ImageEncodingTxtpb, nil + return MessageEncodingTxtpb, nil + case formatYAML: + return MessageEncodingYAML, nil default: - return 0, fmt.Errorf("invalid format for image: %q", format) + return 0, fmt.Errorf("invalid format for message: %q", format) } } @@ -601,3 +642,13 @@ func assumeModuleOrDir(path string) (string, error) { // cannot be parsed into a module, assume dir for here return formatDir, nil } + +type messageRefParserOptions struct { + defaultMessageEncoding MessageEncoding +} + +func newMessageRefParserOptions() *messageRefParserOptions { + return &messageRefParserOptions{ + defaultMessageEncoding: MessageEncodingBinpb, + } +} diff --git a/private/buf/buffetch/ref_parser_test.go b/private/buf/buffetch/ref_parser_test.go index b00477de52..1e116240a3 100644 --- a/private/buf/buffetch/ref_parser_test.go +++ b/private/buf/buffetch/ref_parser_test.go @@ -475,6 +475,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.bin", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.bin", ) @@ -485,6 +486,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.bin.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.bin.gz", ) @@ -495,6 +497,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.binpb", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.binpb", ) @@ -505,6 +508,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.binpb.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.binpb.gz", ) @@ -515,6 +519,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.json", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.json", ) @@ -525,6 +530,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.json.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.json.gz", ) @@ -535,6 +541,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.json.gz", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.json.gz#compression=none", ) @@ -545,6 +552,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.json.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.json.gz#compression=gzip", ) @@ -555,6 +563,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.txtpb", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.txtpb", ) @@ -565,6 +574,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.txtpb.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.txtpb.gz", ) @@ -575,6 +585,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.txtpb.gz", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.txtpb.gz#compression=none", ) @@ -585,9 +596,90 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.txtpb.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.txtpb.gz#compression=gzip", ) + testGetParsedRefSuccess( + t, + internal.NewDirectParsedSingleRef( + formatYAML, + "path/to/file.yaml", + internal.FileSchemeLocal, + internal.CompressionTypeNone, + nil, + ), + "path/to/file.yaml", + ) + testGetParsedRefSuccess( + t, + internal.NewDirectParsedSingleRef( + formatYAML, + "path/to/file.yaml.gz", + internal.FileSchemeLocal, + internal.CompressionTypeGzip, + nil, + ), + "path/to/file.yaml.gz", + ) + testGetParsedRefSuccess( + t, + internal.NewDirectParsedSingleRef( + formatYAML, + "path/to/file.yaml.gz", + internal.FileSchemeLocal, + internal.CompressionTypeNone, + nil, + ), + "path/to/file.yaml.gz#compression=none", + ) + testGetParsedRefSuccess( + t, + internal.NewDirectParsedSingleRef( + formatYAML, + "path/to/file.yaml.gz", + internal.FileSchemeLocal, + internal.CompressionTypeGzip, + nil, + ), + "path/to/file.yaml.gz#compression=gzip", + ) + testGetParsedRefSuccess( + t, + internal.NewDirectParsedSingleRef( + formatYAML, + "path/to/file.yaml", + internal.FileSchemeLocal, + internal.CompressionTypeNone, + map[string]string{ + "use_proto_names": "true", + }, + ), + "path/to/file.yaml#use_proto_names=true", + ) + testGetParsedRefError( + t, + internal.NewOptionsInvalidKeysError("use_something_else"), + "path/to/file.yaml#use_something_else=true", + ) + testGetParsedRefSuccess( + t, + internal.NewDirectParsedSingleRef( + formatJSON, + "path/to/file.json", + internal.FileSchemeLocal, + internal.CompressionTypeNone, + map[string]string{ + "use_proto_names": "true", + }, + ), + "path/to/file.json#use_proto_names=true", + ) + testGetParsedRefError( + t, + internal.NewOptionsInvalidKeysError("use_something_else"), + "path/to/file.json#use_something_else=true", + ) testGetParsedRefSuccess( t, internal.NewDirectParsedSingleRef( @@ -595,6 +687,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "", internal.FileSchemeStdio, internal.CompressionTypeNone, + nil, ), "-", ) @@ -605,6 +698,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "", internal.FileSchemeStdio, internal.CompressionTypeNone, + nil, ), "-#format=json", ) @@ -615,9 +709,21 @@ func TestGetParsedRefSuccess(t *testing.T) { "", internal.FileSchemeStdio, internal.CompressionTypeNone, + nil, ), "-#format=txtpb", ) + testGetParsedRefSuccess( + t, + internal.NewDirectParsedSingleRef( + formatYAML, + "", + internal.FileSchemeStdio, + internal.CompressionTypeNone, + nil, + ), + "-#format=yaml", + ) testGetParsedRefSuccess( t, internal.NewDirectParsedSingleRef( @@ -625,6 +731,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "", internal.FileSchemeNull, internal.CompressionTypeNone, + nil, ), app.DevNullFilePath, ) @@ -635,6 +742,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/dir#format=bin", ) @@ -645,6 +753,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/dir#format=bin,compression=none", ) @@ -655,6 +764,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/dir#format=bin,compression=gzip", ) @@ -665,6 +775,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/dir#format=binpb", ) @@ -675,6 +786,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/dir#format=binpb,compression=none", ) @@ -685,6 +797,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/dir#format=binpb,compression=gzip", ) @@ -929,6 +1042,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file", internal.FileSchemeLocal, internal.CompressionTypeZstd, + nil, ), "path/to/file#format=bin,compression=zstd", ) @@ -939,6 +1053,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.bin.zst", internal.FileSchemeLocal, internal.CompressionTypeZstd, + nil, ), "path/to/file.bin.zst", ) @@ -949,6 +1064,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file", internal.FileSchemeLocal, internal.CompressionTypeZstd, + nil, ), "path/to/file#format=binpb,compression=zstd", ) @@ -959,6 +1075,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.binpb.zst", internal.FileSchemeLocal, internal.CompressionTypeZstd, + nil, ), "path/to/file.binpb.zst", ) @@ -997,6 +1114,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "github.com/path/to/file.bin", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://github.com/path/to/file.bin", ) @@ -1007,6 +1125,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "github.com/path/to/file.ext", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://github.com/path/to/file.ext#format=bin", ) @@ -1017,6 +1136,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "gitlab.com/api/v4/projects/foo/packages/generic/proto/0.0.1/proto.bin?private_token=bar", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://gitlab.com/api/v4/projects/foo/packages/generic/proto/0.0.1/proto.bin?private_token=bar#format=bin", ) @@ -1027,6 +1147,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "github.com/path/to/file.binpb", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://github.com/path/to/file.binpb", ) @@ -1037,6 +1158,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "github.com/path/to/file.ext", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://github.com/path/to/file.ext#format=binpb", ) @@ -1047,6 +1169,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "gitlab.com/api/v4/projects/foo/packages/generic/proto/0.0.1/proto.binpb?private_token=bar", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://gitlab.com/api/v4/projects/foo/packages/generic/proto/0.0.1/proto.binpb?private_token=bar#format=binpb", ) @@ -1121,7 +1244,7 @@ func TestGetParsedRefError(t *testing.T) { ) testGetParsedRefError( t, - internal.NewOptionsInvalidKeyError("foo"), + internal.NewOptionsInvalidKeysError("foo"), "path/to/foo.tar.gz#foo=bar", ) testGetParsedRefError( diff --git a/private/buf/buffetch/ref_parser_unix_test.go b/private/buf/buffetch/ref_parser_unix_test.go index 6b72856608..6127b6a28b 100644 --- a/private/buf/buffetch/ref_parser_unix_test.go +++ b/private/buf/buffetch/ref_parser_unix_test.go @@ -33,6 +33,7 @@ func TestGetParsedRefSuccess_UnixOnly(t *testing.T) { "", internal.FileSchemeStdin, internal.CompressionTypeNone, + nil, ), app.DevStdinFilePath, ) @@ -43,6 +44,7 @@ func TestGetParsedRefSuccess_UnixOnly(t *testing.T) { "", internal.FileSchemeStdout, internal.CompressionTypeNone, + nil, ), app.DevStdoutFilePath, ) diff --git a/private/buf/buffetch/writer.go b/private/buf/buffetch/writer.go index 72d608ac89..f714edba54 100644 --- a/private/buf/buffetch/writer.go +++ b/private/buf/buffetch/writer.go @@ -39,10 +39,10 @@ func newWriter( } } -func (w *writer) PutImageFile( +func (w *writer) PutMessageFile( ctx context.Context, container app.EnvStdoutContainer, - imageRef ImageRef, + messageRef MessageRef, ) (io.WriteCloser, error) { - return w.internalWriter.PutFile(ctx, container, imageRef.internalFileRef()) + return w.internalWriter.PutFile(ctx, container, messageRef.internalSingleRef()) } diff --git a/private/buf/bufref/bufref.go b/private/buf/bufref/bufref.go deleted file mode 100644 index ee4e7aae53..0000000000 --- a/private/buf/bufref/bufref.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufref - -import ( - "strings" -) - -// GetRawPathAndOptions returns the raw path and options from the value provided, -// the rawPath will be non-empty when returning without error here. -func GetRawPathAndOptions(value string) (string, map[string]string, error) { - value = strings.TrimSpace(value) - if value == "" { - return "", nil, newValueEmptyError() - } - - switch splitValue := strings.Split(value, "#"); len(splitValue) { - case 1: - return value, nil, nil - case 2: - path := strings.TrimSpace(splitValue[0]) - optionsString := strings.TrimSpace(splitValue[1]) - if path == "" { - return "", nil, newValueStartsWithHashtagError(value) - } - if optionsString == "" { - return "", nil, newValueEndsWithHashtagError(value) - } - options := make(map[string]string) - for _, pair := range strings.Split(optionsString, ",") { - split := strings.Split(pair, "=") - if len(split) != 2 { - return "", nil, newOptionsInvalidError(optionsString) - } - key := strings.TrimSpace(split[0]) - value := strings.TrimSpace(split[1]) - if key == "" || value == "" { - return "", nil, newOptionsInvalidError(optionsString) - } - if _, ok := options[key]; ok { - return "", nil, newOptionsDuplicateKeyError(key) - } - options[key] = value - } - return path, options, nil - default: - return "", nil, newValueMultipleHashtagsError(value) - } -} diff --git a/private/buf/bufref/errors.go b/private/buf/bufref/errors.go deleted file mode 100644 index 6eabfc9fc2..0000000000 --- a/private/buf/bufref/errors.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufref - -import ( - "errors" - "fmt" -) - -func newValueEmptyError() error { - return errors.New("required") -} - -func newValueMultipleHashtagsError(value string) error { - return fmt.Errorf("%q has multiple #s which is invalid", value) -} - -func newValueStartsWithHashtagError(value string) error { - return fmt.Errorf("%q starts with # which is invalid", value) -} - -func newValueEndsWithHashtagError(value string) error { - return fmt.Errorf("%q ends with # which is invalid", value) -} - -func newOptionsInvalidError(s string) error { - return fmt.Errorf("invalid options: %q", s) -} - -func newOptionsDuplicateKeyError(key string) error { - return fmt.Errorf("duplicate options key: %q", key) -} diff --git a/private/buf/bufref/usage.gen.go b/private/buf/bufref/usage.gen.go deleted file mode 100644 index d5fbc2021b..0000000000 --- a/private/buf/bufref/usage.gen.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Generated. DO NOT EDIT. - -package bufref - -import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/bufwire/bufwire.go b/private/buf/bufwire/bufwire.go index 003d3a2c76..0109377c51 100644 --- a/private/buf/bufwire/bufwire.go +++ b/private/buf/bufwire/bufwire.go @@ -20,7 +20,6 @@ package bufwire import ( "context" - "github.com/bufbuild/buf/private/buf/bufconvert" "github.com/bufbuild/buf/private/buf/buffetch" "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufconfig" @@ -161,7 +160,7 @@ type ImageReader interface { GetImage( ctx context.Context, container app.EnvStdinContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, externalDirOrFilePaths []string, externalExcludeDirOrFilePaths []string, externalDirOrFilePathsAllowNotExist bool, @@ -172,7 +171,7 @@ type ImageReader interface { // NewImageReader returns a new ImageReader. func NewImageReader( logger *zap.Logger, - fetchReader buffetch.ImageReader, + fetchReader buffetch.MessageReader, ) ImageReader { return newImageReader( logger, @@ -189,7 +188,7 @@ type ImageWriter interface { PutImage( ctx context.Context, container app.EnvStdoutContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, image bufimage.Image, asFileDescriptorSet bool, excludeImports bool, @@ -210,23 +209,23 @@ func NewImageWriter( // ProtoEncodingReader is a reader that reads a protobuf message in different encoding. type ProtoEncodingReader interface { // GetMessage reads the message by the messageRef. - // - // Currently, this support bin and JSON format. GetMessage( ctx context.Context, container app.EnvStdinContainer, image bufimage.Image, typeName string, - messageRef bufconvert.MessageEncodingRef, + messageRef buffetch.MessageRef, ) (proto.Message, error) } // NewProtoEncodingReader returns a new ProtoEncodingReader. func NewProtoEncodingReader( logger *zap.Logger, + fetchReader buffetch.MessageReader, ) ProtoEncodingReader { return newProtoEncodingReader( logger, + fetchReader, ) } @@ -241,15 +240,17 @@ type ProtoEncodingWriter interface { container app.EnvStdoutContainer, image bufimage.Image, message proto.Message, - messageRef bufconvert.MessageEncodingRef, + messageRef buffetch.MessageRef, ) error } // NewProtoEncodingWriter returns a new ProtoEncodingWriter. func NewProtoEncodingWriter( logger *zap.Logger, + fetchWriter buffetch.Writer, ) ProtoEncodingWriter { return newProtoEncodingWriter( logger, + fetchWriter, ) } diff --git a/private/buf/bufwire/file_lister.go b/private/buf/bufwire/file_lister.go index 7e8cee376b..82c98d035e 100644 --- a/private/buf/bufwire/file_lister.go +++ b/private/buf/bufwire/file_lister.go @@ -171,7 +171,7 @@ func (e *fileLister) listFilesWithoutImports( } } return fileInfos, nil, nil - case buffetch.ImageRef: + case buffetch.MessageRef: // if we have an image, list the files in the image image, err := e.imageReader.GetImage( ctx, diff --git a/private/buf/bufwire/image_config_reader.go b/private/buf/bufwire/image_config_reader.go index 5b7fee41c8..00e5c86055 100644 --- a/private/buf/bufwire/image_config_reader.go +++ b/private/buf/bufwire/image_config_reader.go @@ -78,7 +78,7 @@ func (i *imageConfigReader) GetImageConfigs( excludeSourceCodeInfo bool, ) ([]ImageConfig, []bufanalysis.FileAnnotation, error) { switch t := ref.(type) { - case buffetch.ImageRef: + case buffetch.MessageRef: env, err := i.getImageImageConfig( ctx, container, @@ -193,7 +193,7 @@ func (i *imageConfigReader) getSourceOrModuleImageConfigs( func (i *imageConfigReader) getImageImageConfig( ctx context.Context, container app.EnvStdinContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, configOverride string, externalDirOrFilePaths []string, externalExcludeDirOrFilePaths []string, @@ -203,7 +203,7 @@ func (i *imageConfigReader) getImageImageConfig( image, err := i.imageReader.GetImage( ctx, container, - imageRef, + messageRef, externalDirOrFilePaths, externalExcludeDirOrFilePaths, externalDirOrFilePathsAllowNotExist, diff --git a/private/buf/bufwire/image_reader.go b/private/buf/bufwire/image_reader.go index 11191f7614..2ddc318e88 100644 --- a/private/buf/bufwire/image_reader.go +++ b/private/buf/bufwire/image_reader.go @@ -38,13 +38,13 @@ const ( type imageReader struct { logger *zap.Logger - fetchReader buffetch.ImageReader + fetchReader buffetch.MessageReader tracer trace.Tracer } func newImageReader( logger *zap.Logger, - fetchReader buffetch.ImageReader, + fetchReader buffetch.MessageReader, ) *imageReader { return &imageReader{ logger: logger.Named(loggerName), @@ -56,7 +56,7 @@ func newImageReader( func (i *imageReader) GetImage( ctx context.Context, container app.EnvStdinContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, externalDirOrFilePaths []string, externalExcludeDirOrFilePaths []string, externalDirOrFilePathsAllowNotExist bool, @@ -70,7 +70,7 @@ func (i *imageReader) GetImage( span.SetStatus(codes.Error, retErr.Error()) } }() - readCloser, err := i.fetchReader.GetImageFile(ctx, container, imageRef) + readCloser, err := i.fetchReader.GetMessageFile(ctx, container, messageRef) if err != nil { return nil, err } @@ -83,11 +83,11 @@ func (i *imageReader) GetImage( } protoImage := &imagev1.Image{} var imageFromProtoOptions []bufimage.NewImageForProtoOption - switch imageEncoding := imageRef.ImageEncoding(); imageEncoding { + switch messageEncoding := messageRef.MessageEncoding(); messageEncoding { // we have to double parse due to custom options // See https://github.com/golang/protobuf/issues/1123 // TODO: revisit - case buffetch.ImageEncodingBin: + case buffetch.MessageEncodingBinpb: _, span := i.tracer.Start(ctx, "wire_unmarshal") if err := protoencoding.NewWireUnmarshaler(nil).Unmarshal(data, protoImage); err != nil { span.RecordError(err) @@ -96,7 +96,7 @@ func (i *imageReader) GetImage( return nil, fmt.Errorf("could not unmarshal image: %v", err) } span.End() - case buffetch.ImageEncodingJSON: + case buffetch.MessageEncodingJSON: resolver, err := i.bootstrapResolver(ctx, protoencoding.NewJSONUnmarshaler(nil), data) if err != nil { return nil, err @@ -111,7 +111,7 @@ func (i *imageReader) GetImage( jsonUnmarshalSpan.End() // we've already re-parsed, by unmarshalling 2x above imageFromProtoOptions = append(imageFromProtoOptions, bufimage.WithNoReparse()) - case buffetch.ImageEncodingTxtpb: + case buffetch.MessageEncodingTxtpb: resolver, err := i.bootstrapResolver(ctx, protoencoding.NewTxtpbUnmarshaler(nil), data) if err != nil { return nil, err @@ -126,8 +126,23 @@ func (i *imageReader) GetImage( txtpbUnmarshalSpan.End() // we've already re-parsed, by unmarshalling 2x above imageFromProtoOptions = append(imageFromProtoOptions, bufimage.WithNoReparse()) + case buffetch.MessageEncodingYAML: + resolver, err := i.bootstrapResolver(ctx, protoencoding.NewYAMLUnmarshaler(nil), data) + if err != nil { + return nil, err + } + _, yamlUnmarshalSpan := i.tracer.Start(ctx, "yaml_unmarshal") + if err := protoencoding.NewYAMLUnmarshaler(resolver).Unmarshal(data, protoImage); err != nil { + yamlUnmarshalSpan.RecordError(err) + yamlUnmarshalSpan.SetStatus(codes.Error, err.Error()) + yamlUnmarshalSpan.End() + return nil, fmt.Errorf("could not unmarshal image: %v", err) + } + yamlUnmarshalSpan.End() + // we've already re-parsed, by unmarshalling 2x above + imageFromProtoOptions = append(imageFromProtoOptions, bufimage.WithNoReparse()) default: - return nil, fmt.Errorf("unknown image encoding: %v", imageEncoding) + return nil, fmt.Errorf("unknown message encoding: %v", messageEncoding) } if excludeSourceCodeInfo { for _, fileDescriptorProto := range protoImage.File { @@ -143,7 +158,7 @@ func (i *imageReader) GetImage( } imagePaths := make([]string, len(externalDirOrFilePaths)) for i, externalDirOrFilePath := range externalDirOrFilePaths { - imagePath, err := imageRef.PathForExternalPath(externalDirOrFilePath) + imagePath, err := messageRef.PathForExternalPath(externalDirOrFilePath) if err != nil { return nil, err } @@ -151,7 +166,7 @@ func (i *imageReader) GetImage( } excludePaths := make([]string, len(externalExcludeDirOrFilePaths)) for i, excludeDirOrFilePath := range externalExcludeDirOrFilePaths { - excludePath, err := imageRef.PathForExternalPath(excludeDirOrFilePath) + excludePath, err := messageRef.PathForExternalPath(excludeDirOrFilePath) if err != nil { return nil, err } diff --git a/private/buf/bufwire/image_writer.go b/private/buf/bufwire/image_writer.go index 6901882b13..55fd3751d8 100644 --- a/private/buf/bufwire/image_writer.go +++ b/private/buf/bufwire/image_writer.go @@ -47,7 +47,7 @@ func newImageWriter( func (i *imageWriter) PutImage( ctx context.Context, container app.EnvStdoutContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, image bufimage.Image, asFileDescriptorSet bool, excludeImports bool, @@ -61,7 +61,7 @@ func (i *imageWriter) PutImage( } }() // stop short for performance - if imageRef.IsNull() { + if messageRef.IsNull() { return nil } writeImage := image @@ -74,11 +74,11 @@ func (i *imageWriter) PutImage( } else { message = bufimage.ImageToProtoImage(writeImage) } - data, err := i.imageMarshal(ctx, message, image, imageRef.ImageEncoding()) + data, err := i.imageMarshal(ctx, message, image, messageRef) if err != nil { return err } - writeCloser, err := i.fetchWriter.PutImageFile(ctx, container, imageRef) + writeCloser, err := i.fetchWriter.PutMessageFile(ctx, container, messageRef) if err != nil { return err } @@ -93,7 +93,7 @@ func (i *imageWriter) imageMarshal( ctx context.Context, message proto.Message, image bufimage.Image, - imageEncoding buffetch.ImageEncoding, + messageRef buffetch.MessageRef, ) (_ []byte, retErr error) { _, span := otel.GetTracerProvider().Tracer("bufbuild/buf").Start(ctx, "image_marshal") defer span.End() @@ -103,10 +103,10 @@ func (i *imageWriter) imageMarshal( span.SetStatus(codes.Error, retErr.Error()) } }() - switch imageEncoding { - case buffetch.ImageEncodingBin: + switch messageEncoding := messageRef.MessageEncoding(); messageEncoding { + case buffetch.MessageEncodingBinpb: return protoencoding.NewWireMarshaler().Marshal(message) - case buffetch.ImageEncodingJSON: + case buffetch.MessageEncodingJSON: // TODO: verify that image is complete resolver, err := protoencoding.NewResolver( bufimage.ImageToFileDescriptorProtos(image)..., @@ -114,8 +114,8 @@ func (i *imageWriter) imageMarshal( if err != nil { return nil, err } - return protoencoding.NewJSONMarshaler(resolver).Marshal(message) - case buffetch.ImageEncodingTxtpb: + return newJSONMarshaler(resolver, messageRef).Marshal(message) + case buffetch.MessageEncodingTxtpb: // TODO: verify that image is complete resolver, err := protoencoding.NewResolver( bufimage.ImageToFileDescriptorProtos(image)..., @@ -124,7 +124,17 @@ func (i *imageWriter) imageMarshal( return nil, err } return protoencoding.NewTxtpbMarshaler(resolver).Marshal(message) + case buffetch.MessageEncodingYAML: + resolver, err := protoencoding.NewResolver( + bufimage.ImageToFileDescriptorProtos( + image, + )..., + ) + if err != nil { + return nil, err + } + return newYAMLMarshaler(resolver, messageRef).Marshal(message) default: - return nil, fmt.Errorf("unknown image encoding: %v", imageEncoding) + return nil, fmt.Errorf("unknown message encoding: %v", messageEncoding) } } diff --git a/private/buf/bufwire/module_config_reader_test.go b/private/buf/bufwire/module_config_reader_test.go index 65d7f2a7a8..d86c8ffd4a 100644 --- a/private/buf/bufwire/module_config_reader_test.go +++ b/private/buf/bufwire/module_config_reader_test.go @@ -200,10 +200,10 @@ func (r *fakeModuleFetcher) GetModule( ) } -func (r *fakeModuleFetcher) GetImageFile( +func (r *fakeModuleFetcher) GetMessageFile( ctx context.Context, container app.EnvStdinContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, ) (io.ReadCloser, error) { return nil, nil } diff --git a/private/buf/bufwire/proto_encoding_reader.go b/private/buf/bufwire/proto_encoding_reader.go index 8127899c0f..c7e1516e50 100644 --- a/private/buf/bufwire/proto_encoding_reader.go +++ b/private/buf/bufwire/proto_encoding_reader.go @@ -19,9 +19,8 @@ import ( "errors" "fmt" "io" - "os" - "github.com/bufbuild/buf/private/buf/bufconvert" + "github.com/bufbuild/buf/private/buf/buffetch" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufreflect" "github.com/bufbuild/buf/private/pkg/app" @@ -34,16 +33,19 @@ import ( ) type protoEncodingReader struct { - logger *zap.Logger + logger *zap.Logger + fetchReader buffetch.MessageReader } var _ ProtoEncodingReader = &protoEncodingReader{} func newProtoEncodingReader( logger *zap.Logger, + fetchReader buffetch.MessageReader, ) *protoEncodingReader { return &protoEncodingReader{ - logger: logger, + logger: logger, + fetchReader: fetchReader, } } @@ -52,7 +54,7 @@ func (p *protoEncodingReader) GetMessage( container app.EnvStdinContainer, image bufimage.Image, typeName string, - messageRef bufconvert.MessageEncodingRef, + messageRef buffetch.MessageRef, ) (_ proto.Message, retErr error) { ctx, span := otel.GetTracerProvider().Tracer("bufbuild/buf").Start(ctx, "get_message") defer span.End() @@ -71,22 +73,23 @@ func (p *protoEncodingReader) GetMessage( } var unmarshaler protoencoding.Unmarshaler switch messageRef.MessageEncoding() { - case bufconvert.MessageEncodingBinpb: + case buffetch.MessageEncodingBinpb: unmarshaler = protoencoding.NewWireUnmarshaler(resolver) - case bufconvert.MessageEncodingJSON: + case buffetch.MessageEncodingJSON: unmarshaler = protoencoding.NewJSONUnmarshaler(resolver) - case bufconvert.MessageEncodingTxtpb: + case buffetch.MessageEncodingTxtpb: unmarshaler = protoencoding.NewTxtpbUnmarshaler(resolver) + case buffetch.MessageEncodingYAML: + unmarshaler = protoencoding.NewYAMLUnmarshaler( + resolver, + protoencoding.YAMLUnmarshalerWithPath(messageRef.Path()), + ) default: return nil, errors.New("unknown message encoding type") } - readCloser := io.NopCloser(container.Stdin()) - if messageRef.Path() != "-" { - var err error - readCloser, err = os.Open(messageRef.Path()) - if err != nil { - return nil, err - } + readCloser, err := p.fetchReader.GetMessageFile(ctx, container, messageRef) + if err != nil { + return nil, err } defer func() { retErr = multierr.Append(retErr, readCloser.Close()) diff --git a/private/buf/bufwire/proto_encoding_writer.go b/private/buf/bufwire/proto_encoding_writer.go index 1c95eaf29e..5613f29a64 100644 --- a/private/buf/bufwire/proto_encoding_writer.go +++ b/private/buf/bufwire/proto_encoding_writer.go @@ -17,12 +17,10 @@ package bufwire import ( "context" "errors" - "os" - "github.com/bufbuild/buf/private/buf/bufconvert" + "github.com/bufbuild/buf/private/buf/buffetch" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/pkg/app" - "github.com/bufbuild/buf/private/pkg/ioextended" "github.com/bufbuild/buf/private/pkg/protoencoding" "go.uber.org/multierr" "go.uber.org/zap" @@ -30,16 +28,19 @@ import ( ) type protoEncodingWriter struct { - logger *zap.Logger + logger *zap.Logger + fetchWriter buffetch.Writer } var _ ProtoEncodingWriter = &protoEncodingWriter{} func newProtoEncodingWriter( logger *zap.Logger, + fetchWriter buffetch.Writer, ) *protoEncodingWriter { return &protoEncodingWriter{ - logger: logger, + logger: logger, + fetchWriter: fetchWriter, } } @@ -48,7 +49,7 @@ func (p *protoEncodingWriter) PutMessage( container app.EnvStdoutContainer, image bufimage.Image, message proto.Message, - messageRef bufconvert.MessageEncodingRef, + messageRef buffetch.MessageRef, ) (retErr error) { // Currently, this support binpb and JSON format. resolver, err := protoencoding.NewResolver( @@ -59,12 +60,14 @@ func (p *protoEncodingWriter) PutMessage( } var marshaler protoencoding.Marshaler switch messageRef.MessageEncoding() { - case bufconvert.MessageEncodingBinpb: + case buffetch.MessageEncodingBinpb: marshaler = protoencoding.NewWireMarshaler() - case bufconvert.MessageEncodingJSON: - marshaler = protoencoding.NewJSONMarshaler(resolver) - case bufconvert.MessageEncodingTxtpb: + case buffetch.MessageEncodingJSON: + marshaler = newJSONMarshaler(resolver, messageRef) + case buffetch.MessageEncodingTxtpb: marshaler = protoencoding.NewTxtpbMarshaler(resolver) + case buffetch.MessageEncodingYAML: + marshaler = newYAMLMarshaler(resolver, messageRef) default: return errors.New("unknown message encoding type") } @@ -72,12 +75,9 @@ func (p *protoEncodingWriter) PutMessage( if err != nil { return err } - writeCloser := ioextended.NopWriteCloser(container.Stdout()) - if messageRef.Path() != "-" { - writeCloser, err = os.Create(messageRef.Path()) - if err != nil { - return err - } + writeCloser, err := p.fetchWriter.PutMessageFile(ctx, container, messageRef) + if err != nil { + return err } defer func() { retErr = multierr.Append(retErr, writeCloser.Close()) diff --git a/private/buf/bufwire/util.go b/private/buf/bufwire/util.go new file mode 100644 index 0000000000..34babfea9b --- /dev/null +++ b/private/buf/bufwire/util.go @@ -0,0 +1,64 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufwire + +import ( + "github.com/bufbuild/buf/private/buf/buffetch" + "github.com/bufbuild/buf/private/pkg/protoencoding" +) + +func newJSONMarshaler( + resolver protoencoding.Resolver, + messageRef buffetch.MessageRef, +) protoencoding.Marshaler { + jsonMarshalerOptions := []protoencoding.JSONMarshalerOption{ + //protoencoding.JSONMarshalerWithIndent(), + } + if messageRef.UseProtoNames() { + jsonMarshalerOptions = append( + jsonMarshalerOptions, + protoencoding.JSONMarshalerWithUseProtoNames(), + ) + } + if messageRef.UseEnumNumbers() { + jsonMarshalerOptions = append( + jsonMarshalerOptions, + protoencoding.JSONMarshalerWithUseEnumNumbers(), + ) + } + return protoencoding.NewJSONMarshaler(resolver, jsonMarshalerOptions...) +} + +func newYAMLMarshaler( + resolver protoencoding.Resolver, + messageRef buffetch.MessageRef, +) protoencoding.Marshaler { + yamlMarshalerOptions := []protoencoding.YAMLMarshalerOption{ + protoencoding.YAMLMarshalerWithIndent(), + } + if messageRef.UseProtoNames() { + yamlMarshalerOptions = append( + yamlMarshalerOptions, + protoencoding.YAMLMarshalerWithUseProtoNames(), + ) + } + if messageRef.UseEnumNumbers() { + yamlMarshalerOptions = append( + yamlMarshalerOptions, + protoencoding.YAMLMarshalerWithUseEnumNumbers(), + ) + } + return protoencoding.NewYAMLMarshaler(resolver, yamlMarshalerOptions...) +} diff --git a/private/buf/cmd/buf/buf_test.go b/private/buf/cmd/buf/buf_test.go index d6e8e34fb1..a45345cad5 100644 --- a/private/buf/cmd/buf/buf_test.go +++ b/private/buf/cmd/buf/buf_test.go @@ -932,6 +932,31 @@ func TestLsFilesImage1(t *testing.T) { ) } +func TestLsFilesImage1_Yaml(t *testing.T) { + t.Parallel() + stdout := bytes.NewBuffer(nil) + testRun( + t, + 0, + nil, + stdout, + "build", + "-o", + "-#format=yaml", + filepath.Join("testdata", "success"), + ) + testRunStdout( + t, + stdout, + 0, + ` + buf/buf.proto + `, + "ls-files", + "-#format=yaml", + ) +} + func TestLsFilesImage2(t *testing.T) { t.Parallel() stdout := bytes.NewBuffer(nil) @@ -2019,6 +2044,23 @@ func TestConvert(t *testing.T) { "-#format=binpb", ) }) + t.Run("stdin-json-payload-to-yaml-with-image", func(t *testing.T) { + t.Parallel() + file, err := os.Open(convertTestDataDir + "/bin_json/payload.json") + require.NoError(t, err) + testRunStdoutFile(t, + file, + 0, + convertTestDataDir+"/bin_json/payload.yaml", + "convert", + "--type=buf.Foo", + convertTestDataDir+"/bin_json/image.yaml", + "--from", + "-#format=json", + "--to", + "-#format=yaml", + ) + }) t.Run("stdin-image-json-to-binpb", func(t *testing.T) { t.Parallel() file, err := os.Open(convertTestDataDir + "/bin_json/image.json") @@ -2035,6 +2077,22 @@ func TestConvert(t *testing.T) { "-#format=binpb", ) }) + t.Run("stdin-image-json-to-yaml", func(t *testing.T) { + t.Parallel() + file, err := os.Open(convertTestDataDir + "/bin_json/image.json") + require.NoError(t, err) + testRunStdoutFile(t, + file, + 0, + convertTestDataDir+"/bin_json/payload.yaml", + "convert", + "--type=buf.Foo", + "-#format=json", + "--from="+convertTestDataDir+"/bin_json/payload.json", + "--to", + "-#format=yaml", + ) + }) t.Run("stdin-image-txtpb-to-binpb", func(t *testing.T) { t.Parallel() file, err := os.Open(convertTestDataDir + "/bin_json/image.txtpb") @@ -2051,6 +2109,22 @@ func TestConvert(t *testing.T) { "-#format=binpb", ) }) + t.Run("stdin-image-yaml-to-binpb", func(t *testing.T) { + t.Parallel() + file, err := os.Open(convertTestDataDir + "/bin_json/image.yaml") + require.NoError(t, err) + testRunStdoutFile(t, + file, + 0, + convertTestDataDir+"/bin_json/payload.binpb", + "convert", + "--type=buf.Foo", + "-#format=yaml", + "--from="+convertTestDataDir+"/bin_json/payload.yaml", + "--to", + "-#format=binpb", + ) + }) } func TestFormat(t *testing.T) { diff --git a/private/buf/cmd/buf/command/alpha/protoc/flags.go b/private/buf/cmd/buf/command/alpha/protoc/flags.go index ece86bb5c1..37f56b3bc1 100644 --- a/private/buf/cmd/buf/command/alpha/protoc/flags.go +++ b/private/buf/cmd/buf/command/alpha/protoc/flags.go @@ -125,7 +125,7 @@ func (f *flagsBuilder) Bind(flagSet *pflag.FlagSet) { "", fmt.Sprintf( `The location to write the FileDescriptorSet. Must be one of format %s.`, - buffetch.ImageFormatsString, + buffetch.MessageFormatsString, ), ) flagSet.StringVar( diff --git a/private/buf/cmd/buf/command/alpha/protoc/protoc.go b/private/buf/cmd/buf/command/alpha/protoc/protoc.go index 8c668dbea3..3711476db0 100644 --- a/private/buf/cmd/buf/command/alpha/protoc/protoc.go +++ b/private/buf/cmd/buf/command/alpha/protoc/protoc.go @@ -239,13 +239,13 @@ func run( if env.Output == "" { return appcmd.NewInvalidArgumentErrorf("required flag %q not set", outputFlagName) } - imageRef, err := buffetch.NewImageRefParser(container.Logger()).GetImageRef(ctx, env.Output) + messageRef, err := buffetch.NewMessageRefParser(container.Logger()).GetMessageRef(ctx, env.Output) if err != nil { return fmt.Errorf("--%s: %v", outputFlagName, err) } return bufcli.NewWireImageWriter(container.Logger()).PutImage(ctx, container, - imageRef, + messageRef, image, true, !env.IncludeImports, diff --git a/private/buf/cmd/buf/command/build/build.go b/private/buf/cmd/buf/command/build/build.go index 25274dfe9b..678964925d 100644 --- a/private/buf/cmd/buf/command/build/build.go +++ b/private/buf/cmd/buf/command/build/build.go @@ -108,7 +108,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { app.DevNullFilePath, fmt.Sprintf( `The output location for the built image. Must be one of format %s`, - buffetch.ImageFormatsString, + buffetch.MessageFormatsString, ), ) flagSet.StringVar( @@ -155,7 +155,7 @@ func run( if err != nil { return err } - imageRef, err := buffetch.NewImageRefParser(container.Logger()).GetImageRef(ctx, flags.Output) + messageRef, err := buffetch.NewMessageRefParser(container.Logger()).GetMessageRef(ctx, flags.Output) if err != nil { return fmt.Errorf("--%s: %v", outputFlagName, err) } @@ -170,7 +170,7 @@ func run( ).PutImage( ctx, container, - imageRef, + messageRef, image, flags.AsFileDescriptorSet, flags.ExcludeImports, diff --git a/private/buf/cmd/buf/command/convert/convert.go b/private/buf/cmd/buf/command/convert/convert.go index 05e5bcc77b..45fbfd9247 100644 --- a/private/buf/cmd/buf/command/convert/convert.go +++ b/private/buf/cmd/buf/command/convert/convert.go @@ -20,22 +20,24 @@ import ( "fmt" "github.com/bufbuild/buf/private/buf/bufcli" - "github.com/bufbuild/buf/private/buf/bufconvert" + "github.com/bufbuild/buf/private/buf/buffetch" "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil" "github.com/bufbuild/buf/private/gen/data/datawkt" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appflag" + "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/spf13/cobra" "github.com/spf13/pflag" ) const ( - errorFormatFlagName = "error-format" - typeFlagName = "type" - fromFlagName = "from" - outputFlagName = "to" + errorFormatFlagName = "error-format" + typeFlagName = "type" + fromFlagName = "from" + outputFlagName = "to" + disableSymlinksFlagName = "disable-symlinks" ) // NewCommand returns a new Command. @@ -90,10 +92,11 @@ Use a module on the bsr: } type flags struct { - ErrorFormat string - Type string - From string - To string + ErrorFormat string + Type string + From string + To string + DisableSymlinks bool // special InputHashtag string @@ -105,6 +108,7 @@ func newFlags() *flags { func (f *flags) Bind(flagSet *pflag.FlagSet) { bufcli.BindInputHashtag(flagSet, &f.InputHashtag) + bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) flagSet.StringVar( &f.ErrorFormat, errorFormatFlagName, @@ -126,7 +130,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { "-", fmt.Sprintf( `The location of the payload to be converted. Supported formats are %s`, - bufconvert.MessageEncodingFormatsString, + buffetch.MessageFormatsString, ), ) flagSet.StringVar( @@ -135,7 +139,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { "-", fmt.Sprintf( `The output location of the conversion. Supported formats are %s`, - bufconvert.MessageEncodingFormatsString, + buffetch.MessageFormatsString, ), ) } @@ -189,12 +193,21 @@ func run( if inputErr != nil && image == nil { return inputErr } - fromMessageRef, err := bufconvert.NewMessageEncodingRef(ctx, flags.From, bufconvert.MessageEncodingBinpb) + fromMessageRef, err := buffetch.NewMessageRefParser( + container.Logger(), + buffetch.MessageRefParserWithDefaultMessageEncoding( + buffetch.MessageEncodingBinpb, + ), + ).GetMessageRef(ctx, flags.From) if err != nil { return fmt.Errorf("--%s: %v", outputFlagName, err) } + storageosProvider := bufcli.NewStorageosProvider(flags.DisableSymlinks) + runner := command.NewRunner() message, err := bufcli.NewWireProtoEncodingReader( container.Logger(), + storageosProvider, + runner, ).GetMessage( ctx, container, @@ -209,7 +222,12 @@ func run( if err != nil { return err } - outputMessageRef, err := bufconvert.NewMessageEncodingRef(ctx, flags.To, defaultToEncoding) + toMessageRef, err := buffetch.NewMessageRefParser( + container.Logger(), + buffetch.MessageRefParserWithDefaultMessageEncoding( + defaultToEncoding, + ), + ).GetMessageRef(ctx, flags.To) if err != nil { return fmt.Errorf("--%s: %v", outputFlagName, err) } @@ -220,20 +238,22 @@ func run( container, image, message, - outputMessageRef, + toMessageRef, ) } // inverseEncoding returns the opposite encoding of the provided encoding, // which will be the default output encoding for a given payload encoding. -func inverseEncoding(encoding bufconvert.MessageEncoding) (bufconvert.MessageEncoding, error) { +func inverseEncoding(encoding buffetch.MessageEncoding) (buffetch.MessageEncoding, error) { switch encoding { - case bufconvert.MessageEncodingBinpb: - return bufconvert.MessageEncodingJSON, nil - case bufconvert.MessageEncodingJSON: - return bufconvert.MessageEncodingBinpb, nil - case bufconvert.MessageEncodingTxtpb: - return bufconvert.MessageEncodingBinpb, nil + case buffetch.MessageEncodingBinpb: + return buffetch.MessageEncodingJSON, nil + case buffetch.MessageEncodingJSON: + return buffetch.MessageEncodingBinpb, nil + case buffetch.MessageEncodingTxtpb: + return buffetch.MessageEncodingBinpb, nil + case buffetch.MessageEncodingYAML: + return buffetch.MessageEncodingBinpb, nil default: return 0, fmt.Errorf("unknown message encoding %v", encoding) } diff --git a/private/buf/cmd/buf/command/convert/convert_test.go b/private/buf/cmd/buf/command/convert/convert_test.go index 6222f9bcb6..b4407579f2 100644 --- a/private/buf/cmd/buf/command/convert/convert_test.go +++ b/private/buf/cmd/buf/command/convert/convert_test.go @@ -74,6 +74,23 @@ func TestConvertDir(t *testing.T) { "-#format=json", ) }) + t.Run("default-input-txtpb", func(t *testing.T) { + t.Parallel() + appcmdtesting.RunCommandExitCodeStdout( + t, + cmd, + 0, + `one: "55"`, + nil, + nil, + "--type", + "buf.Foo", + "--from", + "testdata/convert/bin_json/payload.txtpb", + "--to", + "-#format=yaml", + ) + }) t.Run("from-stdin-bin", func(t *testing.T) { t.Parallel() appcmdtesting.RunCommandExitCodeStdoutStdinFile( @@ -106,7 +123,7 @@ func TestConvertDir(t *testing.T) { "-#format=binpb", ) }) - t.Run("from-stdin-txtpb", func(t *testing.T) { + t.Run("from-stdin-txtpb-json", func(t *testing.T) { t.Parallel() appcmdtesting.RunCommandExitCodeStdoutStdinFile( t, @@ -124,6 +141,24 @@ func TestConvertDir(t *testing.T) { "-#format=json", ) }) + t.Run("from-stdin-txtpb-yaml", func(t *testing.T) { + t.Parallel() + appcmdtesting.RunCommandExitCodeStdoutStdinFile( + t, + cmd, + 0, + `one: "55"`, + nil, + "testdata/convert/bin_json/payload.txtpb", + + "--type", + "buf.Foo", + "--from", + "-#format=txtpb", + "--to", + "-#format=yaml", + ) + }) t.Run("discarded-stdin", func(t *testing.T) { t.Parallel() appcmdtesting.RunCommandExitCodeStdout( @@ -186,6 +221,23 @@ func TestConvertDir(t *testing.T) { "-#format=json", ) }) + t.Run("wellknowntype-txtpb", func(t *testing.T) { + t.Parallel() + appcmdtesting.RunCommandExitCodeStdout( + t, + cmd, + 0, + `3600s`, + nil, + nil, + "--type", + "google.protobuf.Duration", + "--from", + "testdata/convert/bin_json/duration.txtpb", + "--to", + "-#format=yaml", + ) + }) t.Run("wellknowntype-format-bin", func(t *testing.T) { t.Parallel() appcmdtesting.RunCommandExitCodeStdoutFile( @@ -201,6 +253,21 @@ func TestConvertDir(t *testing.T) { "-#format=bin", ) }) + t.Run("wellknowntype-format-bin", func(t *testing.T) { + t.Parallel() + appcmdtesting.RunCommandExitCodeStdoutFile( + t, + cmd, + 0, + "testdata/convert/bin_json/duration.bin", + nil, + nil, + "--type=google.protobuf.Duration", + "--from=testdata/convert/bin_json/duration.yaml", + "--to", + "-#format=bin", + ) + }) t.Run("wellknowntype-format-binpb", func(t *testing.T) { t.Parallel() appcmdtesting.RunCommandExitCodeStdoutFile( @@ -216,6 +283,21 @@ func TestConvertDir(t *testing.T) { "-#format=binpb", ) }) + t.Run("wellknowntype-format-binpb", func(t *testing.T) { + t.Parallel() + appcmdtesting.RunCommandExitCodeStdoutFile( + t, + cmd, + 0, + "testdata/convert/bin_json/duration.binpb", + nil, + nil, + "--type=google.protobuf.Duration", + "--from=testdata/convert/bin_json/duration.yaml", + "--to", + "-#format=binpb", + ) + }) t.Run("wellknowntype-incorrect-input", func(t *testing.T) { t.Parallel() appcmdtesting.RunCommandExitCodeStdout( diff --git a/private/buf/cmd/buf/command/convert/testdata/convert/bin_json/duration.yaml b/private/buf/cmd/buf/command/convert/testdata/convert/bin_json/duration.yaml new file mode 100644 index 0000000000..a59699000a --- /dev/null +++ b/private/buf/cmd/buf/command/convert/testdata/convert/bin_json/duration.yaml @@ -0,0 +1 @@ +3600s diff --git a/private/buf/cmd/buf/command/convert/testdata/convert/bin_json/image.yaml b/private/buf/cmd/buf/command/convert/testdata/convert/bin_json/image.yaml new file mode 100644 index 0000000000..95dfc141b6 --- /dev/null +++ b/private/buf/cmd/buf/command/convert/testdata/convert/bin_json/image.yaml @@ -0,0 +1,89 @@ +file: + - bufExtension: + isImport: false + isSyntaxUnspecified: false + messageType: + - field: + - jsonName: one + label: LABEL_OPTIONAL + name: one + number: 1 + type: TYPE_INT64 + name: Foo + name: buf.proto + package: buf + sourceCodeInfo: + location: + - span: + - 0 + - 0 + - 6 + - 1 + - path: + - 12 + span: + - 0 + - 0 + - 18 + - path: + - 2 + span: + - 2 + - 0 + - 12 + - path: + - 4 + - 0 + span: + - 4 + - 0 + - 6 + - 1 + - path: + - 4 + - 0 + - 1 + span: + - 4 + - 8 + - 11 + - path: + - 4 + - 0 + - 2 + - 0 + span: + - 5 + - 2 + - 16 + - path: + - 4 + - 0 + - 2 + - 0 + - 5 + span: + - 5 + - 2 + - 7 + - path: + - 4 + - 0 + - 2 + - 0 + - 1 + span: + - 5 + - 8 + - 11 + - path: + - 4 + - 0 + - 2 + - 0 + - 3 + span: + - 5 + - 14 + - 15 + syntax: proto3 diff --git a/private/buf/cmd/buf/command/convert/testdata/convert/bin_json/payload.yaml b/private/buf/cmd/buf/command/convert/testdata/convert/bin_json/payload.yaml new file mode 100644 index 0000000000..bf8bc328bb --- /dev/null +++ b/private/buf/cmd/buf/command/convert/testdata/convert/bin_json/payload.yaml @@ -0,0 +1 @@ +one: "55" diff --git a/private/buf/cmd/protoc-gen-buf-breaking/breaking.go b/private/buf/cmd/protoc-gen-buf-breaking/breaking.go index e0dfeb7c1c..268f7e452f 100644 --- a/private/buf/cmd/protoc-gen-buf-breaking/breaking.go +++ b/private/buf/cmd/protoc-gen-buf-breaking/breaking.go @@ -94,7 +94,7 @@ func handle( if !externalConfig.LimitToInputFiles { files = nil } - againstImageRef, err := buffetch.NewImageRefParser(logger).GetImageRef(ctx, externalConfig.AgainstInput) + againstMessageRef, err := buffetch.NewMessageRefParser(logger).GetMessageRef(ctx, externalConfig.AgainstInput) if err != nil { return fmt.Errorf("against_input: %v", err) } @@ -104,7 +104,7 @@ func handle( againstImage, err := imageReader.GetImage( ctx, newContainer(container), - againstImageRef, + againstMessageRef, files, // limit to the input files if specified nil, // exclude paths are not supported on this plugin true, // allow files in the against input to not exist diff --git a/private/pkg/protoencoding/json_marshaler.go b/private/pkg/protoencoding/json_marshaler.go index 22cba9cd9e..d12c306984 100644 --- a/private/pkg/protoencoding/json_marshaler.go +++ b/private/pkg/protoencoding/json_marshaler.go @@ -26,6 +26,7 @@ type jsonMarshaler struct { resolver Resolver indent string useProtoNames bool + useEnumNumbers bool emitUnpopulated bool } @@ -46,6 +47,7 @@ func (m *jsonMarshaler) Marshal(message proto.Message) ([]byte, error) { options := protojson.MarshalOptions{ Resolver: m.resolver, UseProtoNames: m.useProtoNames, + UseEnumNumbers: m.useEnumNumbers, EmitUnpopulated: m.emitUnpopulated, } data, err := options.Marshal(message) diff --git a/private/pkg/protoencoding/protoencoding.go b/private/pkg/protoencoding/protoencoding.go index 8e9fc99037..f384cf8f90 100644 --- a/private/pkg/protoencoding/protoencoding.go +++ b/private/pkg/protoencoding/protoencoding.go @@ -16,6 +16,7 @@ package protoencoding import ( "github.com/bufbuild/buf/private/pkg/protodescriptor" + "github.com/bufbuild/protoyaml-go" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoreflect" @@ -73,7 +74,7 @@ func NewWireMarshaler() Marshaler { // NewJSONMarshaler returns a new Marshaler for JSON. // // This has the potential to be unstable over time. -// resolver can be nil if unknown and are only needed for extensions. +// resolver can be nil if unknown and is only needed for extensions. func NewJSONMarshaler(resolver Resolver, options ...JSONMarshalerOption) Marshaler { return newJSONMarshaler(resolver, options...) } @@ -88,13 +89,20 @@ func JSONMarshalerWithIndent() JSONMarshalerOption { } } -// JSONMarshalerWithUseProtoNames says to use an use proto names. +// JSONMarshalerWithUseProtoNames says to use proto names. func JSONMarshalerWithUseProtoNames() JSONMarshalerOption { return func(jsonMarshaler *jsonMarshaler) { jsonMarshaler.useProtoNames = true } } +// JSONMarshalerWithUseEnumNumbers says to use enum numbers. +func JSONMarshalerWithUseEnumNumbers() JSONMarshalerOption { + return func(jsonMarshaler *jsonMarshaler) { + jsonMarshaler.useEnumNumbers = true + } +} + // JSONMarshalerWithEmitUnpopulated says to emit unpopulated values func JSONMarshalerWithEmitUnpopulated() JSONMarshalerOption { return func(jsonMarshaler *jsonMarshaler) { @@ -104,11 +112,49 @@ func JSONMarshalerWithEmitUnpopulated() JSONMarshalerOption { // NewTxtpbMarshaler returns a new Marshaler for txtpb. // -// resolver can be nil if unknown and are only needed for extensions. +// resolver can be nil if unknown and is only needed for extensions. func NewTxtpbMarshaler(resolver Resolver) Marshaler { return newTxtpbMarshaler(resolver) } +// NewYAMLMarshaler returns a new Marshaler for YAML. +// +// resolver can be nil if unknown and is only needed for extensions. +func NewYAMLMarshaler(resolver Resolver, options ...YAMLMarshalerOption) Marshaler { + return newYAMLMarshaler(resolver, options...) +} + +// YAMLMarshalerOption is an option for a new YAMLMarshaler. +type YAMLMarshalerOption func(*yamlMarshaler) + +// YAMLMarshalerWithIndent says to use an indent of two spaces. +func YAMLMarshalerWithIndent() YAMLMarshalerOption { + return func(yamlMarshaler *yamlMarshaler) { + yamlMarshaler.indent = 2 + } +} + +// YAMLMarshalerWithUseProtoNames says to use proto names. +func YAMLMarshalerWithUseProtoNames() YAMLMarshalerOption { + return func(yamlMarshaler *yamlMarshaler) { + yamlMarshaler.useProtoNames = true + } +} + +// YAMLMarshalerWithUseEnumNumbers says to use enum numbers. +func YAMLMarshalerWithUseEnumNumbers() YAMLMarshalerOption { + return func(yamlMarshaler *yamlMarshaler) { + yamlMarshaler.useEnumNumbers = true + } +} + +// YAMLMarshalerWithEmitUnpopulated says to emit unpopulated values +func YAMLMarshalerWithEmitUnpopulated() YAMLMarshalerOption { + return func(yamlMarshaler *yamlMarshaler) { + yamlMarshaler.emitUnpopulated = true + } +} + // Unmarshaler unmarshals Messages. type Unmarshaler interface { Unmarshal(data []byte, message proto.Message) error @@ -144,3 +190,26 @@ func JSONUnmarshalerWithDisallowUnknown() JSONUnmarshalerOption { func NewTxtpbUnmarshaler(resolver Resolver) Unmarshaler { return newTxtpbUnmarshaler(resolver) } + +// YAMLUnmarshalerOption is an option for a new YAMLUnmarshaler. +type YAMLUnmarshalerOption func(*yamlUnmarshaler) + +// YAMLUnmarshalerWithPath says to use the given path. +func YAMLUnmarshalerWithPath(path string) YAMLUnmarshalerOption { + return func(yamlUnmarshaler *yamlUnmarshaler) { + yamlUnmarshaler.path = path + } +} + +func YAMLUnmarshalerWithValidator(validator protoyaml.Validator) YAMLUnmarshalerOption { + return func(yamlUnmarshaler *yamlUnmarshaler) { + yamlUnmarshaler.validator = validator + } +} + +// NewYAMLUnmarshaler returns a new Unmarshaler for yaml. +// +// resolver can be nil if unknown and are only needed for extensions. +func NewYAMLUnmarshaler(resolver Resolver, options ...YAMLUnmarshalerOption) Unmarshaler { + return newYAMLUnmarshaler(resolver, options...) +} diff --git a/private/pkg/protoencoding/yaml_marshaler.go b/private/pkg/protoencoding/yaml_marshaler.go new file mode 100644 index 0000000000..e65b4b6271 --- /dev/null +++ b/private/pkg/protoencoding/yaml_marshaler.go @@ -0,0 +1,56 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protoencoding + +import ( + "github.com/bufbuild/protoyaml-go" + "google.golang.org/protobuf/proto" +) + +type yamlMarshaler struct { + resolver Resolver + indent int + useProtoNames bool + useEnumNumbers bool + emitUnpopulated bool +} + +func newYAMLMarshaler(resolver Resolver, options ...YAMLMarshalerOption) Marshaler { + yamlMarshaler := &yamlMarshaler{ + resolver: resolver, + } + for _, option := range options { + option(yamlMarshaler) + } + return yamlMarshaler +} + +func (m *yamlMarshaler) Marshal(message proto.Message) ([]byte, error) { + if err := ReparseUnrecognized(m.resolver, message.ProtoReflect()); err != nil { + return nil, err + } + options := protoyaml.MarshalOptions{ + Indent: m.indent, + Resolver: m.resolver, + UseProtoNames: m.useProtoNames, + UseEnumNumbers: m.useEnumNumbers, + EmitUnpopulated: m.emitUnpopulated, + } + data, err := options.Marshal(message) + if err != nil { + return nil, err + } + return data, nil +} diff --git a/private/pkg/protoencoding/yaml_unmarshaler.go b/private/pkg/protoencoding/yaml_unmarshaler.go new file mode 100644 index 0000000000..a3ac3a5264 --- /dev/null +++ b/private/pkg/protoencoding/yaml_unmarshaler.go @@ -0,0 +1,45 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protoencoding + +import ( + "github.com/bufbuild/protoyaml-go" + "google.golang.org/protobuf/proto" +) + +type yamlUnmarshaler struct { + resolver Resolver + path string + validator protoyaml.Validator +} + +func newYAMLUnmarshaler(resolver Resolver, options ...YAMLUnmarshalerOption) Unmarshaler { + result := &yamlUnmarshaler{ + resolver: resolver, + } + for _, option := range options { + option(result) + } + return result +} + +func (m *yamlUnmarshaler) Unmarshal(data []byte, message proto.Message) error { + options := protoyaml.UnmarshalOptions{ + Resolver: m.resolver, + Validator: m.validator, + Path: m.path, + } + return options.Unmarshal(data, message) +}