src/google/protobuf/wire_format.cc

   1 // Protocol Buffers - Google's data interchange format
   2 // Copyright 2008 Google Inc.  All rights reserved.
   3 // https://developers.google.com/protocol-buffers/
   4 //
   5 // Redistribution and use in source and binary forms, with or without
   6 // modification, are permitted provided that the following conditions are
   7 // met:
   8 //
   9 //     * Redistributions of source code must retain the above copyright
  10 // notice, this list of conditions and the following disclaimer.
  11 //     * Redistributions in binary form must reproduce the above
  12 // copyright notice, this list of conditions and the following disclaimer
  13 // in the documentation and/or other materials provided with the
  14 // distribution.
  15 //     * Neither the name of Google Inc. nor the names of its
  16 // contributors may be used to endorse or promote products derived from
  17 // this software without specific prior written permission.
  18 //
  19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 // Author: kenton@google.com (Kenton Varda)
  32 //  Based on original Protocol Buffers design by
  33 //  Sanjay Ghemawat, Jeff Dean, and others.
  34
  35 #include <stack>
  36 #include <string>
  37 #include <vector>
  38
  39 #include <google/protobuf/wire_format.h>
  40
  41 #include <google/protobuf/stubs/logging.h>
  42 #include <google/protobuf/stubs/common.h>
  43 #include <google/protobuf/stubs/stringprintf.h>
  44 #include <google/protobuf/descriptor.pb.h>
  45 #include <google/protobuf/io/coded_stream.h>
  46 #include <google/protobuf/io/zero_copy_stream.h>
  47 #include <google/protobuf/io/zero_copy_stream_impl.h>
  48 #include <google/protobuf/descriptor.h>
  49 #include <google/protobuf/dynamic_message.h>
  50 #include <google/protobuf/map_field.h>
  51 #include <google/protobuf/map_field_inl.h>
  52 #include <google/protobuf/unknown_field_set.h>
  53
  54
  55 #include <google/protobuf/port_def.inc>
  56
  57 const size_t kMapEntryTagByteSize = 2;
  58
  59 namespace google {
  60 namespace protobuf {
  61 namespace internal {
  62
  63 // Forward declare static functions
  64 static size_t MapKeyDataOnlyByteSize(const FieldDescriptor* field,
  65                                      const MapKey& value);
  66 static size_t MapValueRefDataOnlyByteSize(const FieldDescriptor* field,
  67                                           const MapValueRef& value);
  68
  69 // ===================================================================
  70
  71 bool UnknownFieldSetFieldSkipper::SkipField(io::CodedInputStream* input,
  72                                             uint32 tag) {
  73   return WireFormat::SkipField(input, tag, unknown_fields_);
  74 }
  75
  76 bool UnknownFieldSetFieldSkipper::SkipMessage(io::CodedInputStream* input) {
  77   return WireFormat::SkipMessage(input, unknown_fields_);
  78 }
  79
  80 void UnknownFieldSetFieldSkipper::SkipUnknownEnum(int field_number, int value) {
  81   unknown_fields_->AddVarint(field_number, value);
  82 }
  83
  84 bool WireFormat::SkipField(io::CodedInputStream* input, uint32 tag,
  85                            UnknownFieldSet* unknown_fields) {
  86   int number = WireFormatLite::GetTagFieldNumber(tag);
  87   // Field number 0 is illegal.
  88   if (number == 0) return false;
  89
  90   switch (WireFormatLite::GetTagWireType(tag)) {
  91     case WireFormatLite::WIRETYPE_VARINT: {
  92       uint64 value;
  93       if (!input->ReadVarint64(&value)) return false;
  94       if (unknown_fields != NULL) unknown_fields->AddVarint(number, value);
  95       return true;
  96     }
  97     case WireFormatLite::WIRETYPE_FIXED64: {
  98       uint64 value;
  99       if (!input->ReadLittleEndian64(&value)) return false;
 100       if (unknown_fields != NULL) unknown_fields->AddFixed64(number, value);
 101       return true;
 102     }
 103     case WireFormatLite::WIRETYPE_LENGTH_DELIMITED: {
 104       uint32 length;
 105       if (!input->ReadVarint32(&length)) return false;
 106       if (unknown_fields == NULL) {
 107         if (!input->Skip(length)) return false;
 108       } else {
 109         if (!input->ReadString(unknown_fields->AddLengthDelimited(number),
 110                                length)) {
 111           return false;
 112         }
 113       }
 114       return true;
 115     }
 116     case WireFormatLite::WIRETYPE_START_GROUP: {
 117       if (!input->IncrementRecursionDepth()) return false;
 118       if (!SkipMessage(input, (unknown_fields == NULL)
 119                                   ? NULL
 120                                   : unknown_fields->AddGroup(number))) {
 121         return false;
 122       }
 123       input->DecrementRecursionDepth();
 124       // Check that the ending tag matched the starting tag.
 125       if (!input->LastTagWas(
 126               WireFormatLite::MakeTag(WireFormatLite::GetTagFieldNumber(tag),
 127                                       WireFormatLite::WIRETYPE_END_GROUP))) {
 128         return false;
 129       }
 130       return true;
 131     }
 132     case WireFormatLite::WIRETYPE_END_GROUP: {
 133       return false;
 134     }
 135     case WireFormatLite::WIRETYPE_FIXED32: {
 136       uint32 value;
 137       if (!input->ReadLittleEndian32(&value)) return false;
 138       if (unknown_fields != NULL) unknown_fields->AddFixed32(number, value);
 139       return true;
 140     }
 141     default: {
 142       return false;
 143     }
 144   }
 145 }
 146
 147 bool WireFormat::SkipMessage(io::CodedInputStream* input,
 148                              UnknownFieldSet* unknown_fields) {
 149   while (true) {
 150     uint32 tag = input->ReadTag();
 151     if (tag == 0) {
 152       // End of input.  This is a valid place to end, so return true.
 153       return true;
 154     }
 155
 156     WireFormatLite::WireType wire_type = WireFormatLite::GetTagWireType(tag);
 157
 158     if (wire_type == WireFormatLite::WIRETYPE_END_GROUP) {
 159       // Must be the end of the message.
 160       return true;
 161     }
 162
 163     if (!SkipField(input, tag, unknown_fields)) return false;
 164   }
 165 }
 166
 167 bool WireFormat::ReadPackedEnumPreserveUnknowns(io::CodedInputStream* input,
 168                                                 uint32 field_number,
 169                                                 bool (*is_valid)(int),
 170                                                 UnknownFieldSet* unknown_fields,
 171                                                 RepeatedField<int>* values) {
 172   uint32 length;
 173   if (!input->ReadVarint32(&length)) return false;
 174   io::CodedInputStream::Limit limit = input->PushLimit(length);
 175   while (input->BytesUntilLimit() > 0) {
 176     int value;
 177     if (!WireFormatLite::ReadPrimitive<int, WireFormatLite::TYPE_ENUM>(
 178             input, &value)) {
 179       return false;
 180     }
 181     if (is_valid == NULL || is_valid(value)) {
 182       values->Add(value);
 183     } else {
 184       unknown_fields->AddVarint(field_number, value);
 185     }
 186   }
 187   input->PopLimit(limit);
 188   return true;
 189 }
 190
 191 void WireFormat::SerializeUnknownFields(const UnknownFieldSet& unknown_fields,
 192                                         io::CodedOutputStream* output) {
 193   for (int i = 0; i < unknown_fields.field_count(); i++) {
 194     const UnknownField& field = unknown_fields.field(i);
 195     switch (field.type()) {
 196       case UnknownField::TYPE_VARINT:
 197         output->WriteVarint32(WireFormatLite::MakeTag(
 198             field.number(), WireFormatLite::WIRETYPE_VARINT));
 199         output->WriteVarint64(field.varint());
 200         break;
 201       case UnknownField::TYPE_FIXED32:
 202         output->WriteVarint32(WireFormatLite::MakeTag(
 203             field.number(), WireFormatLite::WIRETYPE_FIXED32));
 204         output->WriteLittleEndian32(field.fixed32());
 205         break;
 206       case UnknownField::TYPE_FIXED64:
 207         output->WriteVarint32(WireFormatLite::MakeTag(
 208             field.number(), WireFormatLite::WIRETYPE_FIXED64));
 209         output->WriteLittleEndian64(field.fixed64());
 210         break;
 211       case UnknownField::TYPE_LENGTH_DELIMITED:
 212         output->WriteVarint32(WireFormatLite::MakeTag(
 213             field.number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED));
 214         output->WriteVarint32(field.length_delimited().size());
 215         output->WriteRawMaybeAliased(field.length_delimited().data(),
 216                                      field.length_delimited().size());
 217         break;
 218       case UnknownField::TYPE_GROUP:
 219         output->WriteVarint32(WireFormatLite::MakeTag(
 220             field.number(), WireFormatLite::WIRETYPE_START_GROUP));
 221         SerializeUnknownFields(field.group(), output);
 222         output->WriteVarint32(WireFormatLite::MakeTag(
 223             field.number(), WireFormatLite::WIRETYPE_END_GROUP));
 224         break;
 225     }
 226   }
 227 }
 228
 229 uint8* WireFormat::SerializeUnknownFieldsToArray(
 230     const UnknownFieldSet& unknown_fields, uint8* target) {
 231   for (int i = 0; i < unknown_fields.field_count(); i++) {
 232     const UnknownField& field = unknown_fields.field(i);
 233
 234     switch (field.type()) {
 235       case UnknownField::TYPE_VARINT:
 236         target = WireFormatLite::WriteInt64ToArray(field.number(),
 237                                                    field.varint(), target);
 238         break;
 239       case UnknownField::TYPE_FIXED32:
 240         target = WireFormatLite::WriteFixed32ToArray(field.number(),
 241                                                      field.fixed32(), target);
 242         break;
 243       case UnknownField::TYPE_FIXED64:
 244         target = WireFormatLite::WriteFixed64ToArray(field.number(),
 245                                                      field.fixed64(), target);
 246         break;
 247       case UnknownField::TYPE_LENGTH_DELIMITED:
 248         target = WireFormatLite::WriteBytesToArray(
 249             field.number(), field.length_delimited(), target);
 250         break;
 251       case UnknownField::TYPE_GROUP:
 252         target = WireFormatLite::WriteTagToArray(
 253             field.number(), WireFormatLite::WIRETYPE_START_GROUP, target);
 254         target = SerializeUnknownFieldsToArray(field.group(), target);
 255         target = WireFormatLite::WriteTagToArray(
 256             field.number(), WireFormatLite::WIRETYPE_END_GROUP, target);
 257         break;
 258     }
 259   }
 260   return target;
 261 }
 262
 263 void WireFormat::SerializeUnknownMessageSetItems(
 264     const UnknownFieldSet& unknown_fields, io::CodedOutputStream* output) {
 265   for (int i = 0; i < unknown_fields.field_count(); i++) {
 266     const UnknownField& field = unknown_fields.field(i);
 267     // The only unknown fields that are allowed to exist in a MessageSet are
 268     // messages, which are length-delimited.
 269     if (field.type() == UnknownField::TYPE_LENGTH_DELIMITED) {
 270       // Start group.
 271       output->WriteVarint32(WireFormatLite::kMessageSetItemStartTag);
 272
 273       // Write type ID.
 274       output->WriteVarint32(WireFormatLite::kMessageSetTypeIdTag);
 275       output->WriteVarint32(field.number());
 276
 277       // Write message.
 278       output->WriteVarint32(WireFormatLite::kMessageSetMessageTag);
 279       field.SerializeLengthDelimitedNoTag(output);
 280
 281       // End group.
 282       output->WriteVarint32(WireFormatLite::kMessageSetItemEndTag);
 283     }
 284   }
 285 }
 286
 287 uint8* WireFormat::SerializeUnknownMessageSetItemsToArray(
 288     const UnknownFieldSet& unknown_fields, uint8* target) {
 289   for (int i = 0; i < unknown_fields.field_count(); i++) {
 290     const UnknownField& field = unknown_fields.field(i);
 291
 292     // The only unknown fields that are allowed to exist in a MessageSet are
 293     // messages, which are length-delimited.
 294     if (field.type() == UnknownField::TYPE_LENGTH_DELIMITED) {
 295       // Start group.
 296       target = io::CodedOutputStream::WriteTagToArray(
 297           WireFormatLite::kMessageSetItemStartTag, target);
 298
 299       // Write type ID.
 300       target = io::CodedOutputStream::WriteTagToArray(
 301           WireFormatLite::kMessageSetTypeIdTag, target);
 302       target =
 303           io::CodedOutputStream::WriteVarint32ToArray(field.number(), target);
 304
 305       // Write message.
 306       target = io::CodedOutputStream::WriteTagToArray(
 307           WireFormatLite::kMessageSetMessageTag, target);
 308       target = field.SerializeLengthDelimitedNoTagToArray(target);
 309
 310       // End group.
 311       target = io::CodedOutputStream::WriteTagToArray(
 312           WireFormatLite::kMessageSetItemEndTag, target);
 313     }
 314   }
 315
 316   return target;
 317 }
 318
 319 size_t WireFormat::ComputeUnknownFieldsSize(
 320     const UnknownFieldSet& unknown_fields) {
 321   size_t size = 0;
 322   for (int i = 0; i < unknown_fields.field_count(); i++) {
 323     const UnknownField& field = unknown_fields.field(i);
 324
 325     switch (field.type()) {
 326       case UnknownField::TYPE_VARINT:
 327         size += io::CodedOutputStream::VarintSize32(WireFormatLite::MakeTag(
 328             field.number(), WireFormatLite::WIRETYPE_VARINT));
 329         size += io::CodedOutputStream::VarintSize64(field.varint());
 330         break;
 331       case UnknownField::TYPE_FIXED32:
 332         size += io::CodedOutputStream::VarintSize32(WireFormatLite::MakeTag(
 333             field.number(), WireFormatLite::WIRETYPE_FIXED32));
 334         size += sizeof(int32);
 335         break;
 336       case UnknownField::TYPE_FIXED64:
 337         size += io::CodedOutputStream::VarintSize32(WireFormatLite::MakeTag(
 338             field.number(), WireFormatLite::WIRETYPE_FIXED64));
 339         size += sizeof(int64);
 340         break;
 341       case UnknownField::TYPE_LENGTH_DELIMITED:
 342         size += io::CodedOutputStream::VarintSize32(WireFormatLite::MakeTag(
 343             field.number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED));
 344         size += io::CodedOutputStream::VarintSize32(
 345             field.length_delimited().size());
 346         size += field.length_delimited().size();
 347         break;
 348       case UnknownField::TYPE_GROUP:
 349         size += io::CodedOutputStream::VarintSize32(WireFormatLite::MakeTag(
 350             field.number(), WireFormatLite::WIRETYPE_START_GROUP));
 351         size += ComputeUnknownFieldsSize(field.group());
 352         size += io::CodedOutputStream::VarintSize32(WireFormatLite::MakeTag(
 353             field.number(), WireFormatLite::WIRETYPE_END_GROUP));
 354         break;
 355     }
 356   }
 357
 358   return size;
 359 }
 360
 361 size_t WireFormat::ComputeUnknownMessageSetItemsSize(
 362     const UnknownFieldSet& unknown_fields) {
 363   size_t size = 0;
 364   for (int i = 0; i < unknown_fields.field_count(); i++) {
 365     const UnknownField& field = unknown_fields.field(i);
 366
 367     // The only unknown fields that are allowed to exist in a MessageSet are
 368     // messages, which are length-delimited.
 369     if (field.type() == UnknownField::TYPE_LENGTH_DELIMITED) {
 370       size += WireFormatLite::kMessageSetItemTagsSize;
 371       size += io::CodedOutputStream::VarintSize32(field.number());
 372
 373       int field_size = field.GetLengthDelimitedSize();
 374       size += io::CodedOutputStream::VarintSize32(field_size);
 375       size += field_size;
 376     }
 377   }
 378
 379   return size;
 380 }
 381
 382 // ===================================================================
 383
 384 bool WireFormat::ParseAndMergePartial(io::CodedInputStream* input,
 385                                       Message* message) {
 386   const Descriptor* descriptor = message->GetDescriptor();
 387   const Reflection* message_reflection = message->GetReflection();
 388
 389   while (true) {
 390     uint32 tag = input->ReadTag();
 391     if (tag == 0) {
 392       // End of input.  This is a valid place to end, so return true.
 393       return true;
 394     }
 395
 396     if (WireFormatLite::GetTagWireType(tag) ==
 397         WireFormatLite::WIRETYPE_END_GROUP) {
 398       // Must be the end of the message.
 399       return true;
 400     }
 401
 402     const FieldDescriptor* field = NULL;
 403
 404     if (descriptor != NULL) {
 405       int field_number = WireFormatLite::GetTagFieldNumber(tag);
 406       field = descriptor->FindFieldByNumber(field_number);
 407
 408       // If that failed, check if the field is an extension.
 409       if (field == NULL && descriptor->IsExtensionNumber(field_number)) {
 410         if (input->GetExtensionPool() == NULL) {
 411           field = message_reflection->FindKnownExtensionByNumber(field_number);
 412         } else {
 413           field = input->GetExtensionPool()->FindExtensionByNumber(
 414               descriptor, field_number);
 415         }
 416       }
 417
 418       // If that failed, but we're a MessageSet, and this is the tag for a
 419       // MessageSet item, then parse that.
 420       if (field == NULL && descriptor->options().message_set_wire_format() &&
 421           tag == WireFormatLite::kMessageSetItemStartTag) {
 422         if (!ParseAndMergeMessageSetItem(input, message)) {
 423           return false;
 424         }
 425         continue;  // Skip ParseAndMergeField(); already taken care of.
 426       }
 427     }
 428
 429     if (!ParseAndMergeField(tag, field, message, input)) {
 430       return false;
 431     }
 432   }
 433 }
 434
 435 bool WireFormat::SkipMessageSetField(io::CodedInputStream* input,
 436                                      uint32 field_number,
 437                                      UnknownFieldSet* unknown_fields) {
 438   uint32 length;
 439   if (!input->ReadVarint32(&length)) return false;
 440   return input->ReadString(unknown_fields->AddLengthDelimited(field_number),
 441                            length);
 442 }
 443
 444 bool WireFormat::ParseAndMergeMessageSetField(uint32 field_number,
 445                                               const FieldDescriptor* field,
 446                                               Message* message,
 447                                               io::CodedInputStream* input) {
 448   const Reflection* message_reflection = message->GetReflection();
 449   if (field == NULL) {
 450     // We store unknown MessageSet extensions as groups.
 451     return SkipMessageSetField(
 452         input, field_number, message_reflection->MutableUnknownFields(message));
 453   } else if (field->is_repeated() ||
 454              field->type() != FieldDescriptor::TYPE_MESSAGE) {
 455     // This shouldn't happen as we only allow optional message extensions to
 456     // MessageSet.
 457     GOOGLE_LOG(ERROR) << "Extensions of MessageSets must be optional messages.";
 458     return false;
 459   } else {
 460     Message* sub_message = message_reflection->MutableMessage(
 461         message, field, input->GetExtensionFactory());
 462     return WireFormatLite::ReadMessage(input, sub_message);
 463   }
 464 }
 465
 466 static bool StrictUtf8Check(const FieldDescriptor* field) {
 467   return field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3;
 468 }
 469
 470 bool WireFormat::ParseAndMergeField(
 471     uint32 tag,
 472     const FieldDescriptor* field,  // May be NULL for unknown
 473     Message* message, io::CodedInputStream* input) {
 474   const Reflection* message_reflection = message->GetReflection();
 475
 476   enum { UNKNOWN, NORMAL_FORMAT, PACKED_FORMAT } value_format;
 477
 478   if (field == NULL) {
 479     value_format = UNKNOWN;
 480   } else if (WireFormatLite::GetTagWireType(tag) ==
 481              WireTypeForFieldType(field->type())) {
 482     value_format = NORMAL_FORMAT;
 483   } else if (field->is_packable() &&
 484              WireFormatLite::GetTagWireType(tag) ==
 485                  WireFormatLite::WIRETYPE_LENGTH_DELIMITED) {
 486     value_format = PACKED_FORMAT;
 487   } else {
 488     // We don't recognize this field. Either the field number is unknown
 489     // or the wire type doesn't match. Put it in our unknown field set.
 490     value_format = UNKNOWN;
 491   }
 492
 493   if (value_format == UNKNOWN) {
 494     return SkipField(input, tag,
 495                      message_reflection->MutableUnknownFields(message));
 496   } else if (value_format == PACKED_FORMAT) {
 497     uint32 length;
 498     if (!input->ReadVarint32(&length)) return false;
 499     io::CodedInputStream::Limit limit = input->PushLimit(length);
 500
 501     switch (field->type()) {
 502 #define HANDLE_PACKED_TYPE(TYPE, CPPTYPE, CPPTYPE_METHOD)                      \
 503   case FieldDescriptor::TYPE_##TYPE: {                                         \
 504     while (input->BytesUntilLimit() > 0) {                                     \
 505       CPPTYPE value;                                                           \
 506       if (!WireFormatLite::ReadPrimitive<CPPTYPE,                              \
 507                                          WireFormatLite::TYPE_##TYPE>(input,   \
 508                                                                       &value)) \
 509         return false;                                                          \
 510       message_reflection->Add##CPPTYPE_METHOD(message, field, value);          \
 511     }                                                                          \
 512     break;                                                                     \
 513   }
 514
 515       HANDLE_PACKED_TYPE(INT32, int32, Int32)
 516       HANDLE_PACKED_TYPE(INT64, int64, Int64)
 517       HANDLE_PACKED_TYPE(SINT32, int32, Int32)
 518       HANDLE_PACKED_TYPE(SINT64, int64, Int64)
 519       HANDLE_PACKED_TYPE(UINT32, uint32, UInt32)
 520       HANDLE_PACKED_TYPE(UINT64, uint64, UInt64)
 521
 522       HANDLE_PACKED_TYPE(FIXED32, uint32, UInt32)
 523       HANDLE_PACKED_TYPE(FIXED64, uint64, UInt64)
 524       HANDLE_PACKED_TYPE(SFIXED32, int32, Int32)
 525       HANDLE_PACKED_TYPE(SFIXED64, int64, Int64)
 526
 527       HANDLE_PACKED_TYPE(FLOAT, float, Float)
 528       HANDLE_PACKED_TYPE(DOUBLE, double, Double)
 529
 530       HANDLE_PACKED_TYPE(BOOL, bool, Bool)
 531 #undef HANDLE_PACKED_TYPE
 532
 533       case FieldDescriptor::TYPE_ENUM: {
 534         while (input->BytesUntilLimit() > 0) {
 535           int value;
 536           if (!WireFormatLite::ReadPrimitive<int, WireFormatLite::TYPE_ENUM>(
 537                   input, &value))
 538             return false;
 539           if (message->GetDescriptor()->file()->syntax() ==
 540               FileDescriptor::SYNTAX_PROTO3) {
 541             message_reflection->AddEnumValue(message, field, value);
 542           } else {
 543             const EnumValueDescriptor* enum_value =
 544                 field->enum_type()->FindValueByNumber(value);
 545             if (enum_value != NULL) {
 546               message_reflection->AddEnum(message, field, enum_value);
 547             } else {
 548               // The enum value is not one of the known values.  Add it to the
 549               // UnknownFieldSet.
 550               int64 sign_extended_value = static_cast<int64>(value);
 551               message_reflection->MutableUnknownFields(message)->AddVarint(
 552                   WireFormatLite::GetTagFieldNumber(tag), sign_extended_value);
 553             }
 554           }
 555         }
 556
 557         break;
 558       }
 559
 560       case FieldDescriptor::TYPE_STRING:
 561       case FieldDescriptor::TYPE_GROUP:
 562       case FieldDescriptor::TYPE_MESSAGE:
 563       case FieldDescriptor::TYPE_BYTES:
 564         // Can't have packed fields of these types: these should be caught by
 565         // the protocol compiler.
 566         return false;
 567         break;
 568     }
 569
 570     input->PopLimit(limit);
 571   } else {
 572     // Non-packed value (value_format == NORMAL_FORMAT)
 573     switch (field->type()) {
 574 #define HANDLE_TYPE(TYPE, CPPTYPE, CPPTYPE_METHOD)                            \
 575   case FieldDescriptor::TYPE_##TYPE: {                                        \
 576     CPPTYPE value;                                                            \
 577     if (!WireFormatLite::ReadPrimitive<CPPTYPE, WireFormatLite::TYPE_##TYPE>( \
 578             input, &value))                                                   \
 579       return false;                                                           \
 580     if (field->is_repeated()) {                                               \
 581       message_reflection->Add##CPPTYPE_METHOD(message, field, value);         \
 582     } else {                                                                  \
 583       message_reflection->Set##CPPTYPE_METHOD(message, field, value);         \
 584     }                                                                         \
 585     break;                                                                    \
 586   }
 587
 588       HANDLE_TYPE(INT32, int32, Int32)
 589       HANDLE_TYPE(INT64, int64, Int64)
 590       HANDLE_TYPE(SINT32, int32, Int32)
 591       HANDLE_TYPE(SINT64, int64, Int64)
 592       HANDLE_TYPE(UINT32, uint32, UInt32)
 593       HANDLE_TYPE(UINT64, uint64, UInt64)
 594
 595       HANDLE_TYPE(FIXED32, uint32, UInt32)
 596       HANDLE_TYPE(FIXED64, uint64, UInt64)
 597       HANDLE_TYPE(SFIXED32, int32, Int32)
 598       HANDLE_TYPE(SFIXED64, int64, Int64)
 599
 600       HANDLE_TYPE(FLOAT, float, Float)
 601       HANDLE_TYPE(DOUBLE, double, Double)
 602
 603       HANDLE_TYPE(BOOL, bool, Bool)
 604 #undef HANDLE_TYPE
 605
 606       case FieldDescriptor::TYPE_ENUM: {
 607         int value;
 608         if (!WireFormatLite::ReadPrimitive<int, WireFormatLite::TYPE_ENUM>(
 609                 input, &value))
 610           return false;
 611         if (field->is_repeated()) {
 612           message_reflection->AddEnumValue(message, field, value);
 613         } else {
 614           message_reflection->SetEnumValue(message, field, value);
 615         }
 616         break;
 617       }
 618
 619       // Handle strings separately so that we can optimize the ctype=CORD case.
 620       case FieldDescriptor::TYPE_STRING: {
 621         bool strict_utf8_check = StrictUtf8Check(field);
 622         std::string value;
 623         if (!WireFormatLite::ReadString(input, &value)) return false;
 624         if (strict_utf8_check) {
 625           if (!WireFormatLite::VerifyUtf8String(value.data(), value.length(),
 626                                                 WireFormatLite::PARSE,
 627                                                 field->full_name().c_str())) {
 628             return false;
 629           }
 630         } else {
 631           VerifyUTF8StringNamedField(value.data(), value.length(), PARSE,
 632                                      field->full_name().c_str());
 633         }
 634         if (field->is_repeated()) {
 635           message_reflection->AddString(message, field, value);
 636         } else {
 637           message_reflection->SetString(message, field, value);
 638         }
 639         break;
 640       }
 641
 642       case FieldDescriptor::TYPE_BYTES: {
 643         std::string value;
 644         if (!WireFormatLite::ReadBytes(input, &value)) return false;
 645         if (field->is_repeated()) {
 646           message_reflection->AddString(message, field, value);
 647         } else {
 648           message_reflection->SetString(message, field, value);
 649         }
 650         break;
 651       }
 652
 653       case FieldDescriptor::TYPE_GROUP: {
 654         Message* sub_message;
 655         if (field->is_repeated()) {
 656           sub_message = message_reflection->AddMessage(
 657               message, field, input->GetExtensionFactory());
 658         } else {
 659           sub_message = message_reflection->MutableMessage(
 660               message, field, input->GetExtensionFactory());
 661         }
 662
 663         if (!WireFormatLite::ReadGroup(WireFormatLite::GetTagFieldNumber(tag),
 664                                        input, sub_message))
 665           return false;
 666         break;
 667       }
 668
 669       case FieldDescriptor::TYPE_MESSAGE: {
 670         Message* sub_message;
 671         if (field->is_repeated()) {
 672           sub_message = message_reflection->AddMessage(
 673               message, field, input->GetExtensionFactory());
 674         } else {
 675           sub_message = message_reflection->MutableMessage(
 676               message, field, input->GetExtensionFactory());
 677         }
 678
 679         if (!WireFormatLite::ReadMessage(input, sub_message)) return false;
 680         break;
 681       }
 682     }
 683   }
 684
 685   return true;
 686 }
 687
 688 bool WireFormat::ParseAndMergeMessageSetItem(io::CodedInputStream* input,
 689                                              Message* message) {
 690   struct MSReflective {
 691     bool ParseField(int type_id, io::CodedInputStream* input) {
 692       const FieldDescriptor* field =
 693           message_reflection->FindKnownExtensionByNumber(type_id);
 694       return ParseAndMergeMessageSetField(type_id, field, message, input);
 695     }
 696
 697     bool SkipField(uint32 tag, io::CodedInputStream* input) {
 698       return WireFormat::SkipField(input, tag, NULL);
 699     }
 700
 701     const Reflection* message_reflection;
 702     Message* message;
 703   };
 704
 705   return ParseMessageSetItemImpl(
 706       input, MSReflective{message->GetReflection(), message});
 707 }
 708
 709 // ===================================================================
 710
 711 void WireFormat::SerializeWithCachedSizes(const Message& message, int size,
 712                                           io::CodedOutputStream* output) {
 713   const Descriptor* descriptor = message.GetDescriptor();
 714   const Reflection* message_reflection = message.GetReflection();
 715   int expected_endpoint = output->ByteCount() + size;
 716
 717   std::vector<const FieldDescriptor*> fields;
 718
 719   // Fields of map entry should always be serialized.
 720   if (descriptor->options().map_entry()) {
 721     for (int i = 0; i < descriptor->field_count(); i++) {
 722       fields.push_back(descriptor->field(i));
 723     }
 724   } else {
 725     message_reflection->ListFields(message, &fields);
 726   }
 727
 728   for (int i = 0; i < fields.size(); i++) {
 729     SerializeFieldWithCachedSizes(fields[i], message, output);
 730   }
 731
 732   if (descriptor->options().message_set_wire_format()) {
 733     SerializeUnknownMessageSetItems(
 734         message_reflection->GetUnknownFields(message), output);
 735   } else {
 736     SerializeUnknownFields(message_reflection->GetUnknownFields(message),
 737                            output);
 738   }
 739
 740   GOOGLE_CHECK_EQ(output->ByteCount(), expected_endpoint)
 741       << ": Protocol message serialized to a size different from what was "
 742          "originally expected.  Perhaps it was modified by another thread "
 743          "during serialization?";
 744 }
 745
 746 static void SerializeMapKeyWithCachedSizes(const FieldDescriptor* field,
 747                                            const MapKey& value,
 748                                            io::CodedOutputStream* output) {
 749   switch (field->type()) {
 750     case FieldDescriptor::TYPE_DOUBLE:
 751     case FieldDescriptor::TYPE_FLOAT:
 752     case FieldDescriptor::TYPE_GROUP:
 753     case FieldDescriptor::TYPE_MESSAGE:
 754     case FieldDescriptor::TYPE_BYTES:
 755     case FieldDescriptor::TYPE_ENUM:
 756       GOOGLE_LOG(FATAL) << "Unsupported";
 757       break;
 758 #define CASE_TYPE(FieldType, CamelFieldType, CamelCppType)                     \
 759   case FieldDescriptor::TYPE_##FieldType:                                      \
 760     WireFormatLite::Write##CamelFieldType(1, value.Get##CamelCppType##Value(), \
 761                                           output);                             \
 762     break;
 763       CASE_TYPE(INT64, Int64, Int64)
 764       CASE_TYPE(UINT64, UInt64, UInt64)
 765       CASE_TYPE(INT32, Int32, Int32)
 766       CASE_TYPE(FIXED64, Fixed64, UInt64)
 767       CASE_TYPE(FIXED32, Fixed32, UInt32)
 768       CASE_TYPE(BOOL, Bool, Bool)
 769       CASE_TYPE(UINT32, UInt32, UInt32)
 770       CASE_TYPE(SFIXED32, SFixed32, Int32)
 771       CASE_TYPE(SFIXED64, SFixed64, Int64)
 772       CASE_TYPE(SINT32, SInt32, Int32)
 773       CASE_TYPE(SINT64, SInt64, Int64)
 774       CASE_TYPE(STRING, String, String)
 775 #undef CASE_TYPE
 776   }
 777 }
 778
 779 static void SerializeMapValueRefWithCachedSizes(const FieldDescriptor* field,
 780                                                 const MapValueRef& value,
 781                                                 io::CodedOutputStream* output) {
 782   switch (field->type()) {
 783 #define CASE_TYPE(FieldType, CamelFieldType, CamelCppType)                     \
 784   case FieldDescriptor::TYPE_##FieldType:                                      \
 785     WireFormatLite::Write##CamelFieldType(2, value.Get##CamelCppType##Value(), \
 786                                           output);                             \
 787     break;
 788     CASE_TYPE(INT64, Int64, Int64)
 789     CASE_TYPE(UINT64, UInt64, UInt64)
 790     CASE_TYPE(INT32, Int32, Int32)
 791     CASE_TYPE(FIXED64, Fixed64, UInt64)
 792     CASE_TYPE(FIXED32, Fixed32, UInt32)
 793     CASE_TYPE(BOOL, Bool, Bool)
 794     CASE_TYPE(UINT32, UInt32, UInt32)
 795     CASE_TYPE(SFIXED32, SFixed32, Int32)
 796     CASE_TYPE(SFIXED64, SFixed64, Int64)
 797     CASE_TYPE(SINT32, SInt32, Int32)
 798     CASE_TYPE(SINT64, SInt64, Int64)
 799     CASE_TYPE(ENUM, Enum, Enum)
 800     CASE_TYPE(DOUBLE, Double, Double)
 801     CASE_TYPE(FLOAT, Float, Float)
 802     CASE_TYPE(STRING, String, String)
 803     CASE_TYPE(BYTES, Bytes, String)
 804     CASE_TYPE(MESSAGE, Message, Message)
 805     CASE_TYPE(GROUP, Group, Message)
 806 #undef CASE_TYPE
 807   }
 808 }
 809
 810 class MapKeySorter {
 811  public:
 812   static std::vector<MapKey> SortKey(const Message& message,
 813                                      const Reflection* reflection,
 814                                      const FieldDescriptor* field) {
 815     std::vector<MapKey> sorted_key_list;
 816     for (MapIterator it =
 817              reflection->MapBegin(const_cast<Message*>(&message), field);
 818          it != reflection->MapEnd(const_cast<Message*>(&message), field);
 819          ++it) {
 820       sorted_key_list.push_back(it.GetKey());
 821     }
 822     MapKeyComparator comparator;
 823     std::sort(sorted_key_list.begin(), sorted_key_list.end(), comparator);
 824     return sorted_key_list;
 825   }
 826
 827  private:
 828   class MapKeyComparator {
 829    public:
 830     bool operator()(const MapKey& a, const MapKey& b) const {
 831       GOOGLE_DCHECK(a.type() == b.type());
 832       switch (a.type()) {
 833 #define CASE_TYPE(CppType, CamelCppType)                                \
 834   case FieldDescriptor::CPPTYPE_##CppType: {                            \
 835     return a.Get##CamelCppType##Value() < b.Get##CamelCppType##Value(); \
 836   }
 837         CASE_TYPE(STRING, String)
 838         CASE_TYPE(INT64, Int64)
 839         CASE_TYPE(INT32, Int32)
 840         CASE_TYPE(UINT64, UInt64)
 841         CASE_TYPE(UINT32, UInt32)
 842         CASE_TYPE(BOOL, Bool)
 843 #undef CASE_TYPE
 844
 845         default:
 846           GOOGLE_LOG(DFATAL) << "Invalid key for map field.";
 847           return true;
 848       }
 849     }
 850   };
 851 };
 852
 853 static void SerializeMapEntry(const FieldDescriptor* field, const MapKey& key,
 854                               const MapValueRef& value,
 855                               io::CodedOutputStream* output) {
 856   const FieldDescriptor* key_field = field->message_type()->field(0);
 857   const FieldDescriptor* value_field = field->message_type()->field(1);
 858
 859   WireFormatLite::WriteTag(field->number(),
 860                            WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output);
 861   size_t size = kMapEntryTagByteSize;
 862   size += MapKeyDataOnlyByteSize(key_field, key);
 863   size += MapValueRefDataOnlyByteSize(value_field, value);
 864   output->WriteVarint32(size);
 865   SerializeMapKeyWithCachedSizes(key_field, key, output);
 866   SerializeMapValueRefWithCachedSizes(value_field, value, output);
 867 }
 868
 869 void WireFormat::SerializeFieldWithCachedSizes(const FieldDescriptor* field,
 870                                                const Message& message,
 871                                                io::CodedOutputStream* output) {
 872   const Reflection* message_reflection = message.GetReflection();
 873
 874   if (field->is_extension() &&
 875       field->containing_type()->options().message_set_wire_format() &&
 876       field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE &&
 877       !field->is_repeated()) {
 878     SerializeMessageSetItemWithCachedSizes(field, message, output);
 879     return;
 880   }
 881
 882   // For map fields, we can use either repeated field reflection or map
 883   // reflection.  Our choice has some subtle effects.  If we use repeated field
 884   // reflection here, then the repeated field representation becomes
 885   // authoritative for this field: any existing references that came from map
 886   // reflection remain valid for reading, but mutations to them are lost and
 887   // will be overwritten next time we call map reflection!
 888   //
 889   // So far this mainly affects Python, which keeps long-term references to map
 890   // values around, and always uses map reflection.  See: b/35918691
 891   //
 892   // Here we choose to use map reflection API as long as the internal
 893   // map is valid. In this way, the serialization doesn't change map field's
 894   // internal state and existing references that came from map reflection remain
 895   // valid for both reading and writing.
 896   if (field->is_map()) {
 897     const MapFieldBase* map_field =
 898         message_reflection->GetMapData(message, field);
 899     if (map_field->IsMapValid()) {
 900       if (output->IsSerializationDeterministic()) {
 901         std::vector<MapKey> sorted_key_list =
 902             MapKeySorter::SortKey(message, message_reflection, field);
 903         for (std::vector<MapKey>::iterator it = sorted_key_list.begin();
 904              it != sorted_key_list.end(); ++it) {
 905           MapValueRef map_value;
 906           message_reflection->InsertOrLookupMapValue(
 907               const_cast<Message*>(&message), field, *it, &map_value);
 908           SerializeMapEntry(field, *it, map_value, output);
 909         }
 910       } else {
 911         for (MapIterator it = message_reflection->MapBegin(
 912                  const_cast<Message*>(&message), field);
 913              it !=
 914              message_reflection->MapEnd(const_cast<Message*>(&message), field);
 915              ++it) {
 916           SerializeMapEntry(field, it.GetKey(), it.GetValueRef(), output);
 917         }
 918       }
 919
 920       return;
 921     }
 922   }
 923
 924   int count = 0;
 925
 926   if (field->is_repeated()) {
 927     count = message_reflection->FieldSize(message, field);
 928   } else if (field->containing_type()->options().map_entry()) {
 929     // Map entry fields always need to be serialized.
 930     count = 1;
 931   } else if (message_reflection->HasField(message, field)) {
 932     count = 1;
 933   }
 934
 935   // map_entries is for maps that'll be deterministically serialized.
 936   std::vector<const Message*> map_entries;
 937   if (count > 1 && field->is_map() && output->IsSerializationDeterministic()) {
 938     map_entries =
 939         DynamicMapSorter::Sort(message, count, message_reflection, field);
 940   }
 941
 942   const bool is_packed = field->is_packed();
 943   if (is_packed && count > 0) {
 944     WireFormatLite::WriteTag(field->number(),
 945                              WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output);
 946     const size_t data_size = FieldDataOnlyByteSize(field, message);
 947     output->WriteVarint32(data_size);
 948   }
 949
 950   for (int j = 0; j < count; j++) {
 951     switch (field->type()) {
 952 #define HANDLE_PRIMITIVE_TYPE(TYPE, CPPTYPE, TYPE_METHOD, CPPTYPE_METHOD)     \
 953   case FieldDescriptor::TYPE_##TYPE: {                                        \
 954     const CPPTYPE value =                                                     \
 955         field->is_repeated()                                                  \
 956             ? message_reflection->GetRepeated##CPPTYPE_METHOD(message, field, \
 957                                                               j)              \
 958             : message_reflection->Get##CPPTYPE_METHOD(message, field);        \
 959     if (is_packed) {                                                          \
 960       WireFormatLite::Write##TYPE_METHOD##NoTag(value, output);               \
 961     } else {                                                                  \
 962       WireFormatLite::Write##TYPE_METHOD(field->number(), value, output);     \
 963     }                                                                         \
 964     break;                                                                    \
 965   }
 966
 967       HANDLE_PRIMITIVE_TYPE(INT32, int32, Int32, Int32)
 968       HANDLE_PRIMITIVE_TYPE(INT64, int64, Int64, Int64)
 969       HANDLE_PRIMITIVE_TYPE(SINT32, int32, SInt32, Int32)
 970       HANDLE_PRIMITIVE_TYPE(SINT64, int64, SInt64, Int64)
 971       HANDLE_PRIMITIVE_TYPE(UINT32, uint32, UInt32, UInt32)
 972       HANDLE_PRIMITIVE_TYPE(UINT64, uint64, UInt64, UInt64)
 973
 974       HANDLE_PRIMITIVE_TYPE(FIXED32, uint32, Fixed32, UInt32)
 975       HANDLE_PRIMITIVE_TYPE(FIXED64, uint64, Fixed64, UInt64)
 976       HANDLE_PRIMITIVE_TYPE(SFIXED32, int32, SFixed32, Int32)
 977       HANDLE_PRIMITIVE_TYPE(SFIXED64, int64, SFixed64, Int64)
 978
 979       HANDLE_PRIMITIVE_TYPE(FLOAT, float, Float, Float)
 980       HANDLE_PRIMITIVE_TYPE(DOUBLE, double, Double, Double)
 981
 982       HANDLE_PRIMITIVE_TYPE(BOOL, bool, Bool, Bool)
 983 #undef HANDLE_PRIMITIVE_TYPE
 984
 985 #define HANDLE_TYPE(TYPE, TYPE_METHOD, CPPTYPE_METHOD)                         \
 986   case FieldDescriptor::TYPE_##TYPE:                                           \
 987     WireFormatLite::Write##TYPE_METHOD(                                        \
 988         field->number(),                                                       \
 989         field->is_repeated()                                                   \
 990             ? (map_entries.empty()                                             \
 991                    ? message_reflection->GetRepeated##CPPTYPE_METHOD(message,  \
 992                                                                      field, j) \
 993                    : *map_entries[j])                                          \
 994             : message_reflection->Get##CPPTYPE_METHOD(message, field),         \
 995         output);                                                               \
 996     break;
 997
 998       HANDLE_TYPE(GROUP, Group, Message)
 999       HANDLE_TYPE(MESSAGE, Message, Message)
1000 #undef HANDLE_TYPE
1001
1002       case FieldDescriptor::TYPE_ENUM: {
1003         const EnumValueDescriptor* value =
1004             field->is_repeated()
1005                 ? message_reflection->GetRepeatedEnum(message, field, j)
1006                 : message_reflection->GetEnum(message, field);
1007         if (is_packed) {
1008           WireFormatLite::WriteEnumNoTag(value->number(), output);
1009         } else {
1010           WireFormatLite::WriteEnum(field->number(), value->number(), output);
1011         }
1012         break;
1013       }
1014
1015       // Handle strings separately so that we can get string references
1016       // instead of copying.
1017       case FieldDescriptor::TYPE_STRING: {
1018         bool strict_utf8_check = StrictUtf8Check(field);
1019         std::string scratch;
1020         const std::string& value =
1021             field->is_repeated()
1022                 ? message_reflection->GetRepeatedStringReference(message, field,
1023                                                                  j, &scratch)
1024                 : message_reflection->GetStringReference(message, field,
1025                                                          &scratch);
1026         if (strict_utf8_check) {
1027           WireFormatLite::VerifyUtf8String(value.data(), value.length(),
1028                                            WireFormatLite::SERIALIZE,
1029                                            field->full_name().c_str());
1030         } else {
1031           VerifyUTF8StringNamedField(value.data(), value.length(), SERIALIZE,
1032                                      field->full_name().c_str());
1033         }
1034         WireFormatLite::WriteString(field->number(), value, output);
1035         break;
1036       }
1037
1038       case FieldDescriptor::TYPE_BYTES: {
1039         std::string scratch;
1040         const std::string& value =
1041             field->is_repeated()
1042                 ? message_reflection->GetRepeatedStringReference(message, field,
1043                                                                  j, &scratch)
1044                 : message_reflection->GetStringReference(message, field,
1045                                                          &scratch);
1046         WireFormatLite::WriteBytes(field->number(), value, output);
1047         break;
1048       }
1049     }
1050   }
1051 }
1052
1053 void WireFormat::SerializeMessageSetItemWithCachedSizes(
1054     const FieldDescriptor* field, const Message& message,
1055     io::CodedOutputStream* output) {
1056   const Reflection* message_reflection = message.GetReflection();
1057
1058   // Start group.
1059   output->WriteVarint32(WireFormatLite::kMessageSetItemStartTag);
1060
1061   // Write type ID.
1062   output->WriteVarint32(WireFormatLite::kMessageSetTypeIdTag);
1063   output->WriteVarint32(field->number());
1064
1065   // Write message.
1066   output->WriteVarint32(WireFormatLite::kMessageSetMessageTag);
1067
1068   const Message& sub_message = message_reflection->GetMessage(message, field);
1069   output->WriteVarint32(sub_message.GetCachedSize());
1070   sub_message.SerializeWithCachedSizes(output);
1071
1072   // End group.
1073   output->WriteVarint32(WireFormatLite::kMessageSetItemEndTag);
1074 }
1075
1076 // ===================================================================
1077
1078 size_t WireFormat::ByteSize(const Message& message) {
1079   const Descriptor* descriptor = message.GetDescriptor();
1080   const Reflection* message_reflection = message.GetReflection();
1081
1082   size_t our_size = 0;
1083
1084   std::vector<const FieldDescriptor*> fields;
1085
1086   // Fields of map entry should always be serialized.
1087   if (descriptor->options().map_entry()) {
1088     for (int i = 0; i < descriptor->field_count(); i++) {
1089       fields.push_back(descriptor->field(i));
1090     }
1091   } else {
1092     message_reflection->ListFields(message, &fields);
1093   }
1094
1095   for (int i = 0; i < fields.size(); i++) {
1096     our_size += FieldByteSize(fields[i], message);
1097   }
1098
1099   if (descriptor->options().message_set_wire_format()) {
1100     our_size += ComputeUnknownMessageSetItemsSize(
1101         message_reflection->GetUnknownFields(message));
1102   } else {
1103     our_size +=
1104         ComputeUnknownFieldsSize(message_reflection->GetUnknownFields(message));
1105   }
1106
1107   return our_size;
1108 }
1109
1110 size_t WireFormat::FieldByteSize(const FieldDescriptor* field,
1111                                  const Message& message) {
1112   const Reflection* message_reflection = message.GetReflection();
1113
1114   if (field->is_extension() &&
1115       field->containing_type()->options().message_set_wire_format() &&
1116       field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE &&
1117       !field->is_repeated()) {
1118     return MessageSetItemByteSize(field, message);
1119   }
1120
1121   size_t count = 0;
1122   if (field->is_repeated()) {
1123     count = FromIntSize(message_reflection->FieldSize(message, field));
1124   } else if (field->containing_type()->options().map_entry()) {
1125     // Map entry fields always need to be serialized.
1126     count = 1;
1127   } else if (message_reflection->HasField(message, field)) {
1128     count = 1;
1129   }
1130
1131   const size_t data_size = FieldDataOnlyByteSize(field, message);
1132   size_t our_size = data_size;
1133   if (field->is_packed()) {
1134     if (data_size > 0) {
1135       // Packed fields get serialized like a string, not their native type.
1136       // Technically this doesn't really matter; the size only changes if it's
1137       // a GROUP
1138       our_size += TagSize(field->number(), FieldDescriptor::TYPE_STRING);
1139       our_size += io::CodedOutputStream::VarintSize32(data_size);
1140     }
1141   } else {
1142     our_size += count * TagSize(field->number(), field->type());
1143   }
1144   return our_size;
1145 }
1146
1147 static size_t MapKeyDataOnlyByteSize(const FieldDescriptor* field,
1148                                      const MapKey& value) {
1149   GOOGLE_DCHECK_EQ(FieldDescriptor::TypeToCppType(field->type()), value.type());
1150   switch (field->type()) {
1151     case FieldDescriptor::TYPE_DOUBLE:
1152     case FieldDescriptor::TYPE_FLOAT:
1153     case FieldDescriptor::TYPE_GROUP:
1154     case FieldDescriptor::TYPE_MESSAGE:
1155     case FieldDescriptor::TYPE_BYTES:
1156     case FieldDescriptor::TYPE_ENUM:
1157       GOOGLE_LOG(FATAL) << "Unsupported";
1158       return 0;
1159 #define CASE_TYPE(FieldType, CamelFieldType, CamelCppType) \
1160   case FieldDescriptor::TYPE_##FieldType:                  \
1161     return WireFormatLite::CamelFieldType##Size(           \
1162         value.Get##CamelCppType##Value());
1163
1164 #define FIXED_CASE_TYPE(FieldType, CamelFieldType) \
1165   case FieldDescriptor::TYPE_##FieldType:          \
1166     return WireFormatLite::k##CamelFieldType##Size;
1167
1168       CASE_TYPE(INT32, Int32, Int32);
1169       CASE_TYPE(INT64, Int64, Int64);
1170       CASE_TYPE(UINT32, UInt32, UInt32);
1171       CASE_TYPE(UINT64, UInt64, UInt64);
1172       CASE_TYPE(SINT32, SInt32, Int32);
1173       CASE_TYPE(SINT64, SInt64, Int64);
1174       CASE_TYPE(STRING, String, String);
1175       FIXED_CASE_TYPE(FIXED32, Fixed32);
1176       FIXED_CASE_TYPE(FIXED64, Fixed64);
1177       FIXED_CASE_TYPE(SFIXED32, SFixed32);
1178       FIXED_CASE_TYPE(SFIXED64, SFixed64);
1179       FIXED_CASE_TYPE(BOOL, Bool);
1180
1181 #undef CASE_TYPE
1182 #undef FIXED_CASE_TYPE
1183   }
1184   GOOGLE_LOG(FATAL) << "Cannot get here";
1185   return 0;
1186 }
1187
1188 static size_t MapValueRefDataOnlyByteSize(const FieldDescriptor* field,
1189                                           const MapValueRef& value) {
1190   switch (field->type()) {
1191     case FieldDescriptor::TYPE_GROUP:
1192       GOOGLE_LOG(FATAL) << "Unsupported";
1193       return 0;
1194 #define CASE_TYPE(FieldType, CamelFieldType, CamelCppType) \
1195   case FieldDescriptor::TYPE_##FieldType:                  \
1196     return WireFormatLite::CamelFieldType##Size(           \
1197         value.Get##CamelCppType##Value());
1198
1199 #define FIXED_CASE_TYPE(FieldType, CamelFieldType) \
1200   case FieldDescriptor::TYPE_##FieldType:          \
1201     return WireFormatLite::k##CamelFieldType##Size;
1202
1203       CASE_TYPE(INT32, Int32, Int32);
1204       CASE_TYPE(INT64, Int64, Int64);
1205       CASE_TYPE(UINT32, UInt32, UInt32);
1206       CASE_TYPE(UINT64, UInt64, UInt64);
1207       CASE_TYPE(SINT32, SInt32, Int32);
1208       CASE_TYPE(SINT64, SInt64, Int64);
1209       CASE_TYPE(STRING, String, String);
1210       CASE_TYPE(BYTES, Bytes, String);
1211       CASE_TYPE(ENUM, Enum, Enum);
1212       CASE_TYPE(MESSAGE, Message, Message);
1213       FIXED_CASE_TYPE(FIXED32, Fixed32);
1214       FIXED_CASE_TYPE(FIXED64, Fixed64);
1215       FIXED_CASE_TYPE(SFIXED32, SFixed32);
1216       FIXED_CASE_TYPE(SFIXED64, SFixed64);
1217       FIXED_CASE_TYPE(DOUBLE, Double);
1218       FIXED_CASE_TYPE(FLOAT, Float);
1219       FIXED_CASE_TYPE(BOOL, Bool);
1220
1221 #undef CASE_TYPE
1222 #undef FIXED_CASE_TYPE
1223   }
1224   GOOGLE_LOG(FATAL) << "Cannot get here";
1225   return 0;
1226 }
1227
1228 size_t WireFormat::FieldDataOnlyByteSize(const FieldDescriptor* field,
1229                                          const Message& message) {
1230   const Reflection* message_reflection = message.GetReflection();
1231
1232   size_t data_size = 0;
1233
1234   if (field->is_map()) {
1235     const MapFieldBase* map_field =
1236         message_reflection->GetMapData(message, field);
1237     if (map_field->IsMapValid()) {
1238       MapIterator iter(const_cast<Message*>(&message), field);
1239       MapIterator end(const_cast<Message*>(&message), field);
1240       const FieldDescriptor* key_field = field->message_type()->field(0);
1241       const FieldDescriptor* value_field = field->message_type()->field(1);
1242       for (map_field->MapBegin(&iter), map_field->MapEnd(&end); iter != end;
1243            ++iter) {
1244         size_t size = kMapEntryTagByteSize;
1245         size += MapKeyDataOnlyByteSize(key_field, iter.GetKey());
1246         size += MapValueRefDataOnlyByteSize(value_field, iter.GetValueRef());
1247         data_size += WireFormatLite::LengthDelimitedSize(size);
1248       }
1249       return data_size;
1250     }
1251   }
1252
1253   size_t count = 0;
1254   if (field->is_repeated()) {
1255     count =
1256         internal::FromIntSize(message_reflection->FieldSize(message, field));
1257   } else if (field->containing_type()->options().map_entry()) {
1258     // Map entry fields always need to be serialized.
1259     count = 1;
1260   } else if (message_reflection->HasField(message, field)) {
1261     count = 1;
1262   }
1263
1264   switch (field->type()) {
1265 #define HANDLE_TYPE(TYPE, TYPE_METHOD, CPPTYPE_METHOD)                      \
1266   case FieldDescriptor::TYPE_##TYPE:                                        \
1267     if (field->is_repeated()) {                                             \
1268       for (int j = 0; j < count; j++) {                                     \
1269         data_size += WireFormatLite::TYPE_METHOD##Size(                     \
1270             message_reflection->GetRepeated##CPPTYPE_METHOD(message, field, \
1271                                                             j));            \
1272       }                                                                     \
1273     } else {                                                                \
1274       data_size += WireFormatLite::TYPE_METHOD##Size(                       \
1275           message_reflection->Get##CPPTYPE_METHOD(message, field));         \
1276     }                                                                       \
1277     break;
1278
1279 #define HANDLE_FIXED_TYPE(TYPE, TYPE_METHOD)                   \
1280   case FieldDescriptor::TYPE_##TYPE:                           \
1281     data_size += count * WireFormatLite::k##TYPE_METHOD##Size; \
1282     break;
1283
1284     HANDLE_TYPE(INT32, Int32, Int32)
1285     HANDLE_TYPE(INT64, Int64, Int64)
1286     HANDLE_TYPE(SINT32, SInt32, Int32)
1287     HANDLE_TYPE(SINT64, SInt64, Int64)
1288     HANDLE_TYPE(UINT32, UInt32, UInt32)
1289     HANDLE_TYPE(UINT64, UInt64, UInt64)
1290
1291     HANDLE_FIXED_TYPE(FIXED32, Fixed32)
1292     HANDLE_FIXED_TYPE(FIXED64, Fixed64)
1293     HANDLE_FIXED_TYPE(SFIXED32, SFixed32)
1294     HANDLE_FIXED_TYPE(SFIXED64, SFixed64)
1295
1296     HANDLE_FIXED_TYPE(FLOAT, Float)
1297     HANDLE_FIXED_TYPE(DOUBLE, Double)
1298
1299     HANDLE_FIXED_TYPE(BOOL, Bool)
1300
1301     HANDLE_TYPE(GROUP, Group, Message)
1302     HANDLE_TYPE(MESSAGE, Message, Message)
1303 #undef HANDLE_TYPE
1304 #undef HANDLE_FIXED_TYPE
1305
1306     case FieldDescriptor::TYPE_ENUM: {
1307       if (field->is_repeated()) {
1308         for (int j = 0; j < count; j++) {
1309           data_size += WireFormatLite::EnumSize(
1310               message_reflection->GetRepeatedEnum(message, field, j)->number());
1311         }
1312       } else {
1313         data_size += WireFormatLite::EnumSize(
1314             message_reflection->GetEnum(message, field)->number());
1315       }
1316       break;
1317     }
1318
1319     // Handle strings separately so that we can get string references
1320     // instead of copying.
1321     case FieldDescriptor::TYPE_STRING:
1322     case FieldDescriptor::TYPE_BYTES: {
1323       for (int j = 0; j < count; j++) {
1324         std::string scratch;
1325         const std::string& value =
1326             field->is_repeated()
1327                 ? message_reflection->GetRepeatedStringReference(message, field,
1328                                                                  j, &scratch)
1329                 : message_reflection->GetStringReference(message, field,
1330                                                          &scratch);
1331         data_size += WireFormatLite::StringSize(value);
1332       }
1333       break;
1334     }
1335   }
1336   return data_size;
1337 }
1338
1339 size_t WireFormat::MessageSetItemByteSize(const FieldDescriptor* field,
1340                                           const Message& message) {
1341   const Reflection* message_reflection = message.GetReflection();
1342
1343   size_t our_size = WireFormatLite::kMessageSetItemTagsSize;
1344
1345   // type_id
1346   our_size += io::CodedOutputStream::VarintSize32(field->number());
1347
1348   // message
1349   const Message& sub_message = message_reflection->GetMessage(message, field);
1350   size_t message_size = sub_message.ByteSizeLong();
1351
1352   our_size += io::CodedOutputStream::VarintSize32(message_size);
1353   our_size += message_size;
1354
1355   return our_size;
1356 }
1357
1358 }  // namespace internal
1359 }  // namespace protobuf
1360 }  // namespace google