Down Integrate Internal Changes (protocolbuffers#6634)

* Down integrate to GitHub

* Remove unintentional double assign

* Change ByteCountInt64 to int64_t

* Revert changes in binary_conformance_test.cc

* Revert unnecessary changes

java/core/src/main/java/com/google/protobuf/AbstractMessage.java

@@ -193,7 +193,7 @@ private static Map convertMapEntryListToMap(List list) {
  if (list.isEmpty()) {
  return Collections.emptyMap();
  }
- Map result = new HashMap();
+ Map result = new HashMap<>();
  Iterator iterator = list.iterator();
  Message entry = (Message) iterator.next();
  Descriptors.Descriptor descriptor = entry.getDescriptorForType();

java/core/src/main/java/com/google/protobuf/ByteString.java

@@ -30,6 +30,10 @@
 
 package com.google.protobuf;
 
+import static com.google.protobuf.TextFormatEscaper.escapeBytes;
+import static java.lang.Integer.toHexString;
+import static java.lang.System.identityHashCode;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -49,6 +53,7 @@
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.NoSuchElementException;
 
 /**
@@ -1268,7 +1273,17 @@ static int checkRange(int startIndex, int endIndex, int size) {
  @Override
  public final String toString() {
  return String.format(
- "<ByteString@%s size=%d>", Integer.toHexString(System.identityHashCode(this)), size());
+ Locale.ROOT,
+ "<ByteString@%s size=%d contents=\"%s\">",
+ toHexString(identityHashCode(this)),
+ size(),
+ truncateAndEscapeForDisplay());
+ }
+
+ private String truncateAndEscapeForDisplay() {
+ final int limit = 50;
+
+ return size() <= limit ? escapeBytes(this) : escapeBytes(substring(0, limit - 3)) + "...";
  }
 
  /**

java/core/src/main/java/com/google/protobuf/FieldSet.java

@@ -526,7 +526,7 @@ private void mergeFromField(final Map.Entry<T, Object> entry) {
  if (descriptor.isRepeated()) {
  Object value = getField(descriptor);
  if (value == null) {
- value = new ArrayList();
+ value = new ArrayList<>();
  }
  for (Object element : (List) otherValue) {
  ((List) value).add(cloneIfMutable(element));
@@ -1266,7 +1266,7 @@ private void mergeFromField(final Map.Entry<T, Object> entry) {
  if (descriptor.isRepeated()) {
  Object value = getField(descriptor);
  if (value == null) {
- value = new ArrayList();
+ value = new ArrayList<>();
  }
  for (Object element : (List) otherValue) {
  ((List) value).add(FieldSet.cloneIfMutable(element));

java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java

@@ -1238,7 +1238,7 @@ public MessageLite getMessageDefaultInstance() {
  Object fromFieldSetType(final Object value) {
  if (descriptor.isRepeated()) {
  if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) {
- final List result = new ArrayList();
+ final List result = new ArrayList<>();
  for (final Object element : (List) value) {
  result.add(singularFromFieldSetType(element));
  }
@@ -1263,7 +1263,7 @@ Object singularFromFieldSetType(final Object value) {
  Object toFieldSetType(final Object value) {
  if (descriptor.isRepeated()) {
  if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) {
- final List result = new ArrayList();
+ final List result = new ArrayList<>();
  for (final Object element : (List) value) {
  result.add(singularToFieldSetType(element));
  }

java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java

@@ -158,7 +158,7 @@ protected void mergeFromAndMakeImmutableInternal(
 
  /**
  * Internal helper to return a modifiable map containing all the fields.
- * The returned Map is modifialbe so that the caller can add additional
+ * The returned Map is modifiable so that the caller can add additional
  * extension fields to implement {@link #getAllFields()}.
  *
  * @param getBytesForString whether to generate ByteString for string fields
@@ -2853,7 +2853,7 @@ private Message coerceType(Message value) {
  @Override
  @SuppressWarnings("unchecked")
  public Object get(GeneratedMessageV3 message) {
- List result = new ArrayList();
+ List result = new ArrayList<>();
  for (int i = 0; i < getRepeatedCount(message); i++) {
  result.add(getRepeated(message, i));
  }
@@ -2863,7 +2863,7 @@ public Object get(GeneratedMessageV3 message) {
  @Override
  @SuppressWarnings("unchecked")
  public Object get(Builder builder) {
- List result = new ArrayList();
+ List result = new ArrayList<>();
  for (int i = 0; i < getRepeatedCount(builder); i++) {
  result.add(getRepeated(builder, i));
  }
@@ -3068,7 +3068,7 @@ private static final class RepeatedEnumFieldAccessor
  @Override
  @SuppressWarnings("unchecked")
  public Object get(final GeneratedMessageV3 message) {
- final List newList = new ArrayList();
+ final List newList = new ArrayList<>();
  final int size = getRepeatedCount(message);
  for (int i = 0; i < size; i++) {
  newList.add(getRepeated(message, i));
@@ -3079,7 +3079,7 @@ public Object get(final GeneratedMessageV3 message) {
  @Override
  @SuppressWarnings("unchecked")
  public Object get(final GeneratedMessageV3.Builder builder) {
- final List newList = new ArrayList();
+ final List newList = new ArrayList<>();
  final int size = getRepeatedCount(builder);
  for (int i = 0; i < size; i++) {
  newList.add(getRepeated(builder, i));

java/core/src/main/java/com/google/protobuf/MapEntry.java

@@ -439,7 +439,7 @@ public UnknownFieldSet getUnknownFields() {
  @Override
  @SuppressWarnings("unchecked")
  public Builder<K, V> clone() {
- return new Builder(metadata, key, value, hasKey, hasValue);
+ return new Builder<>(metadata, key, value, hasKey, hasValue);
  }
  }
 

java/core/src/main/java/com/google/protobuf/MapFieldLite.java

@@ -58,7 +58,7 @@ private MapFieldLite(Map<K, V> mapData) {
  }
 
  @SuppressWarnings({"rawtypes", "unchecked"})
- private static final MapFieldLite EMPTY_MAP_FIELD = new MapFieldLite();
+ private static final MapFieldLite EMPTY_MAP_FIELD = new MapFieldLite<>();
 
  static {
  EMPTY_MAP_FIELD.makeImmutable();

java/core/src/test/java/com/google/protobuf/ByteStringTest.java

@@ -486,6 +486,25 @@ public void testToStringUtf8() {
  "copyToStringUtf8 must respect the charset", testString, byteString.toStringUtf8());
  }
 
+ public void testToString() {
+ String toString =
+ ByteString.copyFrom("Here are some bytes: \t\u00a1".getBytes(Internal.UTF_8)).toString();
+ assertTrue(toString, toString.contains("size=24"));
+ assertTrue(toString, toString.contains("contents=\"Here are some bytes: \\t\\302\\241\""));
+ }
+
+ public void testToString_long() {
+ String toString =
+ ByteString.copyFrom(
+ "123456789012345678901234567890123456789012345678901234567890"
+ .getBytes(Internal.UTF_8))
+ .toString();
+ assertTrue(toString, toString.contains("size=60"));
+ assertTrue(
+ toString,
+ toString.contains("contents=\"12345678901234567890123456789012345678901234567...\""));
+ }
+
  public void testNewOutput_InitialCapacity() throws IOException {
  byte[] bytes = getTestBytes();
  ByteString.Output output = ByteString.newOutput(bytes.length + 100);

java/util/src/main/java/com/google/protobuf/util/JsonFormat.java

@@ -1945,6 +1945,14 @@ private Object parseFieldValue(FieldDescriptor field, JsonElement json, Message.
  return field.getEnumType().findValueByNumber(0);
  }
  return null;
+ } else if (json instanceof JsonObject) {
+ if (field.getType() != FieldDescriptor.Type.MESSAGE
+ && field.getType() != FieldDescriptor.Type.GROUP) {
+ // If the field type is primitive, but the json type is JsonObject rather than
+ // JsonElement, throw a type mismatch error.
+ throw new InvalidProtocolBufferException(
+ String.format("Invalid value: %s for expected type: %s", json, field.getType()));
+ }
  }
  switch (field.getType()) {
  case INT32:

java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java

@@ -1723,6 +1723,22 @@ public int read() throws IOException {
  }
  }
 
+ // Test that an error is thrown if a nested JsonObject is parsed as a primitive field.
+ public void testJsonObjectForPrimitiveField() throws Exception {
+ TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+ try {
+ mergeFromJson(
+ "{\n"
+ + " \"optionalString\": {\n"
+ + " \"invalidNestedString\": \"Hello world\"\n"
+ + " }\n"
+ + "}\n",
+ builder);
+ } catch (InvalidProtocolBufferException e) {
+ // Expected.
+ }
+ }
+
  public void testSortedMapKeys() throws Exception {
  TestMap.Builder mapBuilder = TestMap.newBuilder();
  mapBuilder.putStringToInt32Map("\ud834\udd20", 3); // utf-8 F0 9D 84 A0

js/binary/decoder_test.js

@@ -141,8 +141,14 @@ function doTestSignedValue(readValue,
  }
 
  // Encoding values outside the valid range should assert.
- assertThrows(function() {writeValue.call(encoder, lowerLimit * 1.1);});
- assertThrows(function() {writeValue.call(encoder, upperLimit * 1.1);});
+ var pastLowerLimit = lowerLimit * 1.1;
+ var pastUpperLimit = upperLimit * 1.1;
+ if (pastLowerLimit !== -Infinity) {
+ expect(() => void writeValue.call(encoder, pastLowerLimit)).toThrow();
+ }
+ if (pastUpperLimit !== Infinity) {
+ expect(() => void writeValue.call(encoder, pastUpperLimit)).toThrow();
+ }
 }
 
 describe('binaryDecoderTest', function() {

js/binary/encoder.js

@@ -382,8 +382,10 @@ jspb.BinaryEncoder.prototype.writeInt64String = function(value) {
  * @param {number} value The value to write.
  */
 jspb.BinaryEncoder.prototype.writeFloat = function(value) {
- goog.asserts.assert((value >= -jspb.BinaryConstants.FLOAT32_MAX) &&
- (value <= jspb.BinaryConstants.FLOAT32_MAX));
+ goog.asserts.assert(
+ value === Infinity || value === -Infinity || isNaN(value) ||
+ ((value >= -jspb.BinaryConstants.FLOAT32_MAX) &&
+ (value <= jspb.BinaryConstants.FLOAT32_MAX)));
  jspb.utils.splitFloat32(value);
  this.writeUint32(jspb.utils.split64Low);
 };
@@ -395,8 +397,10 @@ jspb.BinaryEncoder.prototype.writeFloat = function(value) {
  * @param {number} value The value to write.
  */
 jspb.BinaryEncoder.prototype.writeDouble = function(value) {
- goog.asserts.assert((value >= -jspb.BinaryConstants.FLOAT64_MAX) &&
- (value <= jspb.BinaryConstants.FLOAT64_MAX));
+ goog.asserts.assert(
+ value === Infinity || value === -Infinity || isNaN(value) ||
+ ((value >= -jspb.BinaryConstants.FLOAT64_MAX) &&
+ (value <= jspb.BinaryConstants.FLOAT64_MAX)));
  jspb.utils.splitFloat64(value);
  this.writeUint32(jspb.utils.split64Low);
  this.writeUint32(jspb.utils.split64High);

js/binary/utils.js

@@ -255,8 +255,25 @@ jspb.utils.splitFloat64 = function(value) {
  return;
  }
 
- var exp = Math.floor(Math.log(value) / Math.LN2);
- if (exp == 1024) exp = 1023;
+ // Compute the least significant exponent needed to represent the magnitude of
+ // the value by repeadly dividing/multiplying by 2 until the magnitude
+ // crosses 2. While tempting to use log math to find the exponent, at the
+ // bounadaries of precision, the result can be off by one.
+ var maxDoubleExponent = 1023;
+ var minDoubleExponent = -1022;
+ var x = value;
+ var exp = 0;
+ if (x >= 2) {
+ while (x >= 2 && exp < maxDoubleExponent) {
+ exp++;
+ x = x / 2;
+ }
+ } else {
+ while (x < 1 && exp > minDoubleExponent) {
+ x = x * 2;
+ exp--;
+ }
+ }
  var mant = value * Math.pow(2, -exp);
 
  var mantHigh = (mant * jspb.BinaryConstants.TWO_TO_20) & 0xFFFFF;

js/binary/utils_test.js

@@ -341,6 +341,8 @@ describe('binaryUtilsTest', function() {
  var f32_eps = jspb.BinaryConstants.FLOAT32_EPS;
  var f32_min = jspb.BinaryConstants.FLOAT32_MIN;
  var f32_max = jspb.BinaryConstants.FLOAT32_MAX;
+ var f32_max_safe_int = jspb.utils.joinFloat32(0x4b7fffff, 0);
+ var f32_pi = Math.fround(Math.PI);
 
  // NaN.
  jspb.utils.splitFloat32(NaN);
@@ -382,6 +384,13 @@ describe('binaryUtilsTest', function() {
  test(f32_max, 0x7F7FFFFF);
  test(-f32_max, 0xFF7FFFFF);
 
+ // Positive and negative max_safe_int.
+ test(f32_max_safe_int, 0x4B7FFFFF);
+ test(-f32_max_safe_int, 0xCB7FFFFF);
+
+ // Pi.
+ test(f32_pi, 0x40490fdb);
+
  // Various positive values.
  var cursor = f32_eps * 10;
  while (cursor != Infinity) {
@@ -420,10 +429,12 @@ describe('binaryUtilsTest', function() {
  function test(x, opt_highBits, opt_lowBits) {
  jspb.utils.splitFloat64(x);
  if (goog.isDef(opt_highBits)) {
- if (opt_highBits != jspb.utils.split64High) throw 'fail!';
+ var split64High = jspb.utils.split64High;
+ expect(opt_highBits.toString(16)).toEqual(split64High.toString(16));
  }
  if (goog.isDef(opt_lowBits)) {
- if (opt_lowBits != jspb.utils.split64Low) throw 'fail!';
+ var split64Low = jspb.utils.split64Low;
+ expect(opt_lowBits.toString(16)).toEqual(split64Low.toString(16));
  }
  expect(
  jspb.utils.joinFloat64(jspb.utils.split64Low, jspb.utils.split64High))
@@ -438,6 +449,9 @@ describe('binaryUtilsTest', function() {
  test(0, 0x00000000, 0x00000000);
  test(-0, 0x80000000, 0x00000000);
 
+ test(1, 0x3FF00000, 0x00000000);
+ test(2, 0x40000000, 0x00000000);
+
  // Positive and negative epsilon.
  test(f64_eps, 0x00000000, 0x00000001);
  test(-f64_eps, 0x80000000, 0x00000001);
@@ -450,6 +464,19 @@ describe('binaryUtilsTest', function() {
  test(f64_max, 0x7FEFFFFF, 0xFFFFFFFF);
  test(-f64_max, 0xFFEFFFFF, 0xFFFFFFFF);
 
+ test(Number.MAX_SAFE_INTEGER, 0x433FFFFF, 0xFFFFFFFF);
+ test(Number.MIN_SAFE_INTEGER, 0xC33FFFFF, 0xFFFFFFFF);
+
+ // Test various edge cases with mantissa of all 1, all 0, or just the
+ // highest or lowest significant bit.
+ test(4503599627370497, 0x43300000, 0x00000001);
+ test(6755399441055744, 0x43380000, 0x00000000);
+ test(1.348269851146737e+308, 0x7FE80000, 0x00000000);
+ test(1.9999999999999998, 0x3FFFFFFF, 0xFFFFFFFF);
+ test(2.225073858507201e-308, 0x000FFFFF, 0xFFFFFFFF);
+ test(Math.PI, 0x400921fb, 0x54442d18);
+ test(jspb.BinaryConstants.FLOAT32_MIN, 0x38100000, 0x00000000);
+
  // Various positive values.
  var cursor = f64_eps * 10;
  while (cursor != Infinity) {