1 // Go support for Protocol Buffers - Google's data interchange format
3 // Copyright 2010 The Go Authors. All rights reserved.
4 // https://github.com/golang/protobuf
6 // Redistribution and use in source and binary forms, with or without
7 // modification, are permitted provided that the following conditions are
10 // * Redistributions of source code must retain the above copyright
11 // notice, this list of conditions and the following disclaimer.
12 // * Redistributions in binary form must reproduce the above
13 // copyright notice, this list of conditions and the following disclaimer
14 // in the documentation and/or other materials provided with the
16 // * Neither the name of Google Inc. nor the names of its
17 // contributors may be used to endorse or promote products derived from
18 // this software without specific prior written permission.
20 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 * Routines for decoding protocol buffer data to construct in-memory representations.
46 // errOverflow is returned when an integer is too large to be represented.
47 var errOverflow = errors.New("proto: integer overflow")
49 // ErrInternalBadWireType is returned by generated code when an incorrect
50 // wire type is encountered. It does not get returned to user code.
51 var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof")
53 // The fundamental decoders that interpret bytes on the wire.
54 // Those that take integer types all return uint64 and are
55 // therefore of type valueDecoder.
57 // DecodeVarint reads a varint-encoded integer from the slice.
58 // It returns the integer and the number of bytes consumed, or
59 // zero if there is not enough.
60 // This is the format for the
61 // int32, int64, uint32, uint64, bool, and enum
62 // protocol buffer types.
63 func DecodeVarint(buf []byte) (x uint64, n int) {
64 for shift := uint(0); shift < 64; shift += 7 {
70 x |= (b & 0x7F) << shift
76 // The number is too large to represent in a 64-bit value.
80 func (p *Buffer) decodeVarintSlow() (x uint64, err error) {
84 for shift := uint(0); shift < 64; shift += 7 {
86 err = io.ErrUnexpectedEOF
91 x |= (uint64(b) & 0x7F) << shift
98 // The number is too large to represent in a 64-bit value.
103 // DecodeVarint reads a varint-encoded integer from the Buffer.
104 // This is the format for the
105 // int32, int64, uint32, uint64, bool, and enum
106 // protocol buffer types.
107 func (p *Buffer) DecodeVarint() (x uint64, err error) {
112 return 0, io.ErrUnexpectedEOF
113 } else if buf[i] < 0x80 {
115 return uint64(buf[i]), nil
116 } else if len(buf)-i < 10 {
117 return p.decodeVarintSlow()
121 // we already checked the first byte
122 x = uint64(buf[i]) - 0x80
195 // x -= 0x80 << 63 // Always zero.
197 return 0, errOverflow
204 // DecodeFixed64 reads a 64-bit integer from the Buffer.
205 // This is the format for the
206 // fixed64, sfixed64, and double protocol buffer types.
207 func (p *Buffer) DecodeFixed64() (x uint64, err error) {
210 if i < 0 || i > len(p.buf) {
211 err = io.ErrUnexpectedEOF
216 x = uint64(p.buf[i-8])
217 x |= uint64(p.buf[i-7]) << 8
218 x |= uint64(p.buf[i-6]) << 16
219 x |= uint64(p.buf[i-5]) << 24
220 x |= uint64(p.buf[i-4]) << 32
221 x |= uint64(p.buf[i-3]) << 40
222 x |= uint64(p.buf[i-2]) << 48
223 x |= uint64(p.buf[i-1]) << 56
227 // DecodeFixed32 reads a 32-bit integer from the Buffer.
228 // This is the format for the
229 // fixed32, sfixed32, and float protocol buffer types.
230 func (p *Buffer) DecodeFixed32() (x uint64, err error) {
233 if i < 0 || i > len(p.buf) {
234 err = io.ErrUnexpectedEOF
239 x = uint64(p.buf[i-4])
240 x |= uint64(p.buf[i-3]) << 8
241 x |= uint64(p.buf[i-2]) << 16
242 x |= uint64(p.buf[i-1]) << 24
246 // DecodeZigzag64 reads a zigzag-encoded 64-bit integer
248 // This is the format used for the sint64 protocol buffer type.
249 func (p *Buffer) DecodeZigzag64() (x uint64, err error) {
250 x, err = p.DecodeVarint()
254 x = (x >> 1) ^ uint64((int64(x&1)<<63)>>63)
258 // DecodeZigzag32 reads a zigzag-encoded 32-bit integer
260 // This is the format used for the sint32 protocol buffer type.
261 func (p *Buffer) DecodeZigzag32() (x uint64, err error) {
262 x, err = p.DecodeVarint()
266 x = uint64((uint32(x) >> 1) ^ uint32((int32(x&1)<<31)>>31))
270 // These are not ValueDecoders: they produce an array of bytes or a string.
271 // bytes, embedded messages
273 // DecodeRawBytes reads a count-delimited byte buffer from the Buffer.
274 // This is the format used for the bytes protocol buffer
275 // type and for embedded messages.
276 func (p *Buffer) DecodeRawBytes(alloc bool) (buf []byte, err error) {
277 n, err := p.DecodeVarint()
284 return nil, fmt.Errorf("proto: bad byte length %d", nb)
287 if end < p.index || end > len(p.buf) {
288 return nil, io.ErrUnexpectedEOF
292 // todo: check if can get more uses of alloc=false
293 buf = p.buf[p.index:end]
298 buf = make([]byte, nb)
299 copy(buf, p.buf[p.index:])
304 // DecodeStringBytes reads an encoded string from the Buffer.
305 // This is the format used for the proto2 string type.
306 func (p *Buffer) DecodeStringBytes() (s string, err error) {
307 buf, err := p.DecodeRawBytes(false)
311 return string(buf), nil
314 // Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
315 // If the protocol buffer has extensions, and the field matches, add it as an extension.
316 // Otherwise, if the XXX_unrecognized field exists, append the skipped data there.
317 func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base structPointer, unrecField field) error {
320 err := o.skip(t, tag, wire)
325 if !unrecField.IsValid() {
329 ptr := structPointer_Bytes(base, unrecField)
331 // Add the skipped field to struct field
335 o.EncodeVarint(uint64(tag<<3 | wire))
336 *ptr = append(o.buf, obuf[oi:o.index]...)
343 // Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
344 func (o *Buffer) skip(t reflect.Type, tag, wire int) error {
351 _, err = o.DecodeVarint()
353 _, err = o.DecodeFixed64()
355 _, err = o.DecodeRawBytes(false)
357 _, err = o.DecodeFixed32()
360 u, err = o.DecodeVarint()
364 fwire := int(u & 0x7)
365 if fwire == WireEndGroup {
369 err = o.skip(t, ftag, fwire)
375 err = fmt.Errorf("proto: can't skip unknown wire type %d for %s", wire, t)
380 // Unmarshaler is the interface representing objects that can
381 // unmarshal themselves. The method should reset the receiver before
382 // decoding starts. The argument points to data that may be
383 // overwritten, so implementations should not keep references to the
385 type Unmarshaler interface {
386 Unmarshal([]byte) error
389 // Unmarshal parses the protocol buffer representation in buf and places the
390 // decoded result in pb. If the struct underlying pb does not match
391 // the data in buf, the results can be unpredictable.
393 // Unmarshal resets pb before starting to unmarshal, so any
394 // existing data in pb is always removed. Use UnmarshalMerge
395 // to preserve and append to existing data.
396 func Unmarshal(buf []byte, pb Message) error {
398 return UnmarshalMerge(buf, pb)
401 // UnmarshalMerge parses the protocol buffer representation in buf and
402 // writes the decoded result to pb. If the struct underlying pb does not match
403 // the data in buf, the results can be unpredictable.
405 // UnmarshalMerge merges into existing data in pb.
406 // Most code should use Unmarshal instead.
407 func UnmarshalMerge(buf []byte, pb Message) error {
408 // If the object can unmarshal itself, let it.
409 if u, ok := pb.(Unmarshaler); ok {
410 return u.Unmarshal(buf)
412 return NewBuffer(buf).Unmarshal(pb)
415 // DecodeMessage reads a count-delimited message from the Buffer.
416 func (p *Buffer) DecodeMessage(pb Message) error {
417 enc, err := p.DecodeRawBytes(false)
421 return NewBuffer(enc).Unmarshal(pb)
424 // DecodeGroup reads a tag-delimited group from the Buffer.
425 func (p *Buffer) DecodeGroup(pb Message) error {
426 typ, base, err := getbase(pb)
430 return p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), true, base)
433 // Unmarshal parses the protocol buffer representation in the
434 // Buffer and places the decoded result in pb. If the struct
435 // underlying pb does not match the data in the buffer, the results can be
438 // Unlike proto.Unmarshal, this does not reset pb before starting to unmarshal.
439 func (p *Buffer) Unmarshal(pb Message) error {
440 // If the object can unmarshal itself, let it.
441 if u, ok := pb.(Unmarshaler); ok {
442 err := u.Unmarshal(p.buf[p.index:])
447 typ, base, err := getbase(pb)
452 err = p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), false, base)
461 // unmarshalType does the work of unmarshaling a structure.
462 func (o *Buffer) unmarshalType(st reflect.Type, prop *StructProperties, is_group bool, base structPointer) error {
464 required, reqFields := prop.reqCount, uint64(0)
467 for err == nil && o.index < len(o.buf) {
470 u, err = o.DecodeVarint()
475 if wire == WireEndGroup {
478 // Not enough information to determine the exact field.
480 return &RequiredNotSetError{"{Unknown}"}
482 return nil // input is satisfied
484 return fmt.Errorf("proto: %s: wiretype end group for non-group", st)
488 return fmt.Errorf("proto: %s: illegal tag %d (wire type %d)", st, tag, wire)
490 fieldnum, ok := prop.decoderTags.get(tag)
492 // Maybe it's an extension?
494 if e, _ := extendable(structPointer_Interface(base, st)); isExtensionField(e, int32(tag)) {
495 if err = o.skip(st, tag, wire); err == nil {
496 extmap := e.extensionsWrite()
497 ext := extmap[int32(tag)] // may be missing
498 ext.enc = append(ext.enc, o.buf[oi:o.index]...)
499 extmap[int32(tag)] = ext
504 // Maybe it's a oneof?
505 if prop.oneofUnmarshaler != nil {
506 m := structPointer_Interface(base, st).(Message)
507 // First return value indicates whether tag is a oneof field.
508 ok, err = prop.oneofUnmarshaler(m, tag, wire, o)
509 if err == ErrInternalBadWireType {
510 // Map the error to something more descriptive.
511 // Do the formatting here to save generated code space.
512 err = fmt.Errorf("bad wiretype for oneof field in %T", m)
518 err = o.skipAndSave(st, tag, wire, base, prop.unrecField)
521 p := prop.Prop[fieldnum]
524 fmt.Fprintf(os.Stderr, "proto: no protobuf decoder for %s.%s\n", st, st.Field(fieldnum).Name)
528 if wire != WireStartGroup && wire != p.WireType {
529 if wire == WireBytes && p.packedDec != nil {
533 err = fmt.Errorf("proto: bad wiretype for field %s.%s: got wiretype %d, want %d", st, st.Field(fieldnum).Name, wire, p.WireType)
537 decErr := dec(o, p, base)
538 if decErr != nil && !state.shouldContinue(decErr, p) {
541 if err == nil && p.Required {
542 // Successfully decoded a required field.
544 // use bitmap for fields 1-64 to catch field reuse.
545 var mask uint64 = 1 << uint64(tag-1)
546 if reqFields&mask == 0 {
547 // new required field
552 // This is imprecise. It can be fooled by a required field
553 // with a tag > 64 that is encoded twice; that's very rare.
554 // A fully correct implementation would require allocating
555 // a data structure, which we would like to avoid.
562 return io.ErrUnexpectedEOF
564 if state.err != nil {
568 // Not enough information to determine the exact field. If we use extra
569 // CPU, we could determine the field only if the missing required field
570 // has a tag <= 64 and we check reqFields.
571 return &RequiredNotSetError{"{Unknown}"}
577 // Individual type decoders
579 // u is the decoded value,
580 // v is a pointer to the field (pointer) in the struct
582 // Sizes of the pools to allocate inside the Buffer.
583 // The goal is modest amortization and allocation
584 // on at least 16-byte boundaries.
592 func (o *Buffer) dec_bool(p *Properties, base structPointer) error {
593 u, err := p.valDec(o)
597 if len(o.bools) == 0 {
598 o.bools = make([]bool, boolPoolSize)
601 *structPointer_Bool(base, p.field) = &o.bools[0]
602 o.bools = o.bools[1:]
606 func (o *Buffer) dec_proto3_bool(p *Properties, base structPointer) error {
607 u, err := p.valDec(o)
611 *structPointer_BoolVal(base, p.field) = u != 0
616 func (o *Buffer) dec_int32(p *Properties, base structPointer) error {
617 u, err := p.valDec(o)
621 word32_Set(structPointer_Word32(base, p.field), o, uint32(u))
625 func (o *Buffer) dec_proto3_int32(p *Properties, base structPointer) error {
626 u, err := p.valDec(o)
630 word32Val_Set(structPointer_Word32Val(base, p.field), uint32(u))
635 func (o *Buffer) dec_int64(p *Properties, base structPointer) error {
636 u, err := p.valDec(o)
640 word64_Set(structPointer_Word64(base, p.field), o, u)
644 func (o *Buffer) dec_proto3_int64(p *Properties, base structPointer) error {
645 u, err := p.valDec(o)
649 word64Val_Set(structPointer_Word64Val(base, p.field), o, u)
654 func (o *Buffer) dec_string(p *Properties, base structPointer) error {
655 s, err := o.DecodeStringBytes()
659 *structPointer_String(base, p.field) = &s
663 func (o *Buffer) dec_proto3_string(p *Properties, base structPointer) error {
664 s, err := o.DecodeStringBytes()
668 *structPointer_StringVal(base, p.field) = s
672 // Decode a slice of bytes ([]byte).
673 func (o *Buffer) dec_slice_byte(p *Properties, base structPointer) error {
674 b, err := o.DecodeRawBytes(true)
678 *structPointer_Bytes(base, p.field) = b
682 // Decode a slice of bools ([]bool).
683 func (o *Buffer) dec_slice_bool(p *Properties, base structPointer) error {
684 u, err := p.valDec(o)
688 v := structPointer_BoolSlice(base, p.field)
689 *v = append(*v, u != 0)
693 // Decode a slice of bools ([]bool) in packed format.
694 func (o *Buffer) dec_slice_packed_bool(p *Properties, base structPointer) error {
695 v := structPointer_BoolSlice(base, p.field)
697 nn, err := o.DecodeVarint()
701 nb := int(nn) // number of bytes of encoded bools
709 u, err := p.valDec(o)
713 y = append(y, u != 0)
720 // Decode a slice of int32s ([]int32).
721 func (o *Buffer) dec_slice_int32(p *Properties, base structPointer) error {
722 u, err := p.valDec(o)
726 structPointer_Word32Slice(base, p.field).Append(uint32(u))
730 // Decode a slice of int32s ([]int32) in packed format.
731 func (o *Buffer) dec_slice_packed_int32(p *Properties, base structPointer) error {
732 v := structPointer_Word32Slice(base, p.field)
734 nn, err := o.DecodeVarint()
738 nb := int(nn) // number of bytes of encoded int32s
745 u, err := p.valDec(o)
754 // Decode a slice of int64s ([]int64).
755 func (o *Buffer) dec_slice_int64(p *Properties, base structPointer) error {
756 u, err := p.valDec(o)
761 structPointer_Word64Slice(base, p.field).Append(u)
765 // Decode a slice of int64s ([]int64) in packed format.
766 func (o *Buffer) dec_slice_packed_int64(p *Properties, base structPointer) error {
767 v := structPointer_Word64Slice(base, p.field)
769 nn, err := o.DecodeVarint()
773 nb := int(nn) // number of bytes of encoded int64s
780 u, err := p.valDec(o)
789 // Decode a slice of strings ([]string).
790 func (o *Buffer) dec_slice_string(p *Properties, base structPointer) error {
791 s, err := o.DecodeStringBytes()
795 v := structPointer_StringSlice(base, p.field)
800 // Decode a slice of slice of bytes ([][]byte).
801 func (o *Buffer) dec_slice_slice_byte(p *Properties, base structPointer) error {
802 b, err := o.DecodeRawBytes(true)
806 v := structPointer_BytesSlice(base, p.field)
811 // Decode a map field.
812 func (o *Buffer) dec_new_map(p *Properties, base structPointer) error {
813 raw, err := o.DecodeRawBytes(false)
817 oi := o.index // index at the end of this map entry
818 o.index -= len(raw) // move buffer back to start of map entry
820 mptr := structPointer_NewAt(base, p.field, p.mtype) // *map[K]V
821 if mptr.Elem().IsNil() {
822 mptr.Elem().Set(reflect.MakeMap(mptr.Type().Elem()))
824 v := mptr.Elem() // map[K]V
826 // Prepare addressable doubly-indirect placeholders for the key and value types.
827 // See enc_new_map for why.
828 keyptr := reflect.New(reflect.PtrTo(p.mtype.Key())).Elem() // addressable *K
829 keybase := toStructPointer(keyptr.Addr()) // **K
831 var valbase structPointer
832 var valptr reflect.Value
833 switch p.mtype.Elem().Kind() {
837 valptr = reflect.ValueOf(&dummy) // *[]byte
838 valbase = toStructPointer(valptr) // *[]byte
840 // message; valptr is **Msg; need to allocate the intermediate pointer
841 valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V
842 valptr.Set(reflect.New(valptr.Type().Elem()))
843 valbase = toStructPointer(valptr)
846 valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V
847 valbase = toStructPointer(valptr.Addr()) // **V
851 // This parses a restricted wire format, namely the encoding of a message
852 // with two fields. See enc_new_map for the format.
854 // tagcode for key and value properties are always a single byte
855 // because they have tags 1 and 2.
856 tagcode := o.buf[o.index]
859 case p.mkeyprop.tagcode[0]:
860 if err := p.mkeyprop.dec(o, p.mkeyprop, keybase); err != nil {
863 case p.mvalprop.tagcode[0]:
864 if err := p.mvalprop.dec(o, p.mvalprop, valbase); err != nil {
868 // TODO: Should we silently skip this instead?
869 return fmt.Errorf("proto: bad map data tag %d", raw[0])
872 keyelem, valelem := keyptr.Elem(), valptr.Elem()
873 if !keyelem.IsValid() {
874 keyelem = reflect.Zero(p.mtype.Key())
876 if !valelem.IsValid() {
877 valelem = reflect.Zero(p.mtype.Elem())
880 v.SetMapIndex(keyelem, valelem)
885 func (o *Buffer) dec_struct_group(p *Properties, base structPointer) error {
886 bas := structPointer_GetStructPointer(base, p.field)
887 if structPointer_IsNil(bas) {
888 // allocate new nested message
889 bas = toStructPointer(reflect.New(p.stype))
890 structPointer_SetStructPointer(base, p.field, bas)
892 return o.unmarshalType(p.stype, p.sprop, true, bas)
895 // Decode an embedded message.
896 func (o *Buffer) dec_struct_message(p *Properties, base structPointer) (err error) {
897 raw, e := o.DecodeRawBytes(false)
902 bas := structPointer_GetStructPointer(base, p.field)
903 if structPointer_IsNil(bas) {
904 // allocate new nested message
905 bas = toStructPointer(reflect.New(p.stype))
906 structPointer_SetStructPointer(base, p.field, bas)
909 // If the object can unmarshal itself, let it.
911 iv := structPointer_Interface(bas, p.stype)
912 return iv.(Unmarshaler).Unmarshal(raw)
920 err = o.unmarshalType(p.stype, p.sprop, false, bas)
927 // Decode a slice of embedded messages.
928 func (o *Buffer) dec_slice_struct_message(p *Properties, base structPointer) error {
929 return o.dec_slice_struct(p, false, base)
932 // Decode a slice of embedded groups.
933 func (o *Buffer) dec_slice_struct_group(p *Properties, base structPointer) error {
934 return o.dec_slice_struct(p, true, base)
937 // Decode a slice of structs ([]*struct).
938 func (o *Buffer) dec_slice_struct(p *Properties, is_group bool, base structPointer) error {
939 v := reflect.New(p.stype)
940 bas := toStructPointer(v)
941 structPointer_StructPointerSlice(base, p.field).Append(bas)
944 err := o.unmarshalType(p.stype, p.sprop, is_group, bas)
948 raw, err := o.DecodeRawBytes(false)
953 // If the object can unmarshal itself, let it.
956 return iv.(Unmarshaler).Unmarshal(raw)
964 err = o.unmarshalType(p.stype, p.sprop, is_group, bas)