1// Go support for Protocol Buffers - Google's data interchange format 2// 3// Copyright 2010 The Go Authors. All rights reserved. 4// https://github.com/golang/protobuf 5// 6// Redistribution and use in source and binary forms, with or without 7// modification, are permitted provided that the following conditions are 8// met: 9// 10// * Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// * Redistributions in binary form must reproduce the above 13// copyright notice, this list of conditions and the following disclaimer 14// in the documentation and/or other materials provided with the 15// distribution. 16// * Neither the name of Google Inc. nor the names of its 17// contributors may be used to endorse or promote products derived from 18// this software without specific prior written permission. 19// 20// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32package proto 33 34/* 35 * Routines for encoding data into the wire format for protocol buffers. 36 */ 37 38import ( 39 "fmt" 40 "log" 41 "reflect" 42 "sort" 43 "strconv" 44 "strings" 45 "sync" 46) 47 48const debug bool = false 49 50// Constants that identify the encoding of a value on the wire. 51const ( 52 WireVarint = 0 53 WireFixed64 = 1 54 WireBytes = 2 55 WireStartGroup = 3 56 WireEndGroup = 4 57 WireFixed32 = 5 58) 59 60// tagMap is an optimization over map[int]int for typical protocol buffer 61// use-cases. Encoded protocol buffers are often in tag order with small tag 62// numbers. 63type tagMap struct { 64 fastTags []int 65 slowTags map[int]int 66} 67 68// tagMapFastLimit is the upper bound on the tag number that will be stored in 69// the tagMap slice rather than its map. 70const tagMapFastLimit = 1024 71 72func (p *tagMap) get(t int) (int, bool) { 73 if t > 0 && t < tagMapFastLimit { 74 if t >= len(p.fastTags) { 75 return 0, false 76 } 77 fi := p.fastTags[t] 78 return fi, fi >= 0 79 } 80 fi, ok := p.slowTags[t] 81 return fi, ok 82} 83 84func (p *tagMap) put(t int, fi int) { 85 if t > 0 && t < tagMapFastLimit { 86 for len(p.fastTags) < t+1 { 87 p.fastTags = append(p.fastTags, -1) 88 } 89 p.fastTags[t] = fi 90 return 91 } 92 if p.slowTags == nil { 93 p.slowTags = make(map[int]int) 94 } 95 p.slowTags[t] = fi 96} 97 98// StructProperties represents properties for all the fields of a struct. 99// decoderTags and decoderOrigNames should only be used by the decoder. 100type StructProperties struct { 101 Prop []*Properties // properties for each field 102 reqCount int // required count 103 decoderTags tagMap // map from proto tag to struct field number 104 decoderOrigNames map[string]int // map from original name to struct field number 105 order []int // list of struct field numbers in tag order 106 107 // OneofTypes contains information about the oneof fields in this message. 108 // It is keyed by the original name of a field. 109 OneofTypes map[string]*OneofProperties 110} 111 112// OneofProperties represents information about a specific field in a oneof. 113type OneofProperties struct { 114 Type reflect.Type // pointer to generated struct type for this oneof field 115 Field int // struct field number of the containing oneof in the message 116 Prop *Properties 117} 118 119// Implement the sorting interface so we can sort the fields in tag order, as recommended by the spec. 120// See encode.go, (*Buffer).enc_struct. 121 122func (sp *StructProperties) Len() int { return len(sp.order) } 123func (sp *StructProperties) Less(i, j int) bool { 124 return sp.Prop[sp.order[i]].Tag < sp.Prop[sp.order[j]].Tag 125} 126func (sp *StructProperties) Swap(i, j int) { sp.order[i], sp.order[j] = sp.order[j], sp.order[i] } 127 128// Properties represents the protocol-specific behavior of a single struct field. 129type Properties struct { 130 Name string // name of the field, for error messages 131 OrigName string // original name before protocol compiler (always set) 132 JSONName string // name to use for JSON; determined by protoc 133 Wire string 134 WireType int 135 Tag int 136 Required bool 137 Optional bool 138 Repeated bool 139 Packed bool // relevant for repeated primitives only 140 Enum string // set for enum types only 141 proto3 bool // whether this is known to be a proto3 field 142 oneof bool // whether this is a oneof field 143 144 Default string // default value 145 HasDefault bool // whether an explicit default was provided 146 147 stype reflect.Type // set for struct types only 148 sprop *StructProperties // set for struct types only 149 150 mtype reflect.Type // set for map types only 151 MapKeyProp *Properties // set for map types only 152 MapValProp *Properties // set for map types only 153} 154 155// String formats the properties in the protobuf struct field tag style. 156func (p *Properties) String() string { 157 s := p.Wire 158 s += "," 159 s += strconv.Itoa(p.Tag) 160 if p.Required { 161 s += ",req" 162 } 163 if p.Optional { 164 s += ",opt" 165 } 166 if p.Repeated { 167 s += ",rep" 168 } 169 if p.Packed { 170 s += ",packed" 171 } 172 s += ",name=" + p.OrigName 173 if p.JSONName != p.OrigName { 174 s += ",json=" + p.JSONName 175 } 176 if p.proto3 { 177 s += ",proto3" 178 } 179 if p.oneof { 180 s += ",oneof" 181 } 182 if len(p.Enum) > 0 { 183 s += ",enum=" + p.Enum 184 } 185 if p.HasDefault { 186 s += ",def=" + p.Default 187 } 188 return s 189} 190 191// Parse populates p by parsing a string in the protobuf struct field tag style. 192func (p *Properties) Parse(s string) { 193 // "bytes,49,opt,name=foo,def=hello!" 194 fields := strings.Split(s, ",") // breaks def=, but handled below. 195 if len(fields) < 2 { 196 log.Printf("proto: tag has too few fields: %q", s) 197 return 198 } 199 200 p.Wire = fields[0] 201 switch p.Wire { 202 case "varint": 203 p.WireType = WireVarint 204 case "fixed32": 205 p.WireType = WireFixed32 206 case "fixed64": 207 p.WireType = WireFixed64 208 case "zigzag32": 209 p.WireType = WireVarint 210 case "zigzag64": 211 p.WireType = WireVarint 212 case "bytes", "group": 213 p.WireType = WireBytes 214 // no numeric converter for non-numeric types 215 default: 216 log.Printf("proto: tag has unknown wire type: %q", s) 217 return 218 } 219 220 var err error 221 p.Tag, err = strconv.Atoi(fields[1]) 222 if err != nil { 223 return 224 } 225 226outer: 227 for i := 2; i < len(fields); i++ { 228 f := fields[i] 229 switch { 230 case f == "req": 231 p.Required = true 232 case f == "opt": 233 p.Optional = true 234 case f == "rep": 235 p.Repeated = true 236 case f == "packed": 237 p.Packed = true 238 case strings.HasPrefix(f, "name="): 239 p.OrigName = f[5:] 240 case strings.HasPrefix(f, "json="): 241 p.JSONName = f[5:] 242 case strings.HasPrefix(f, "enum="): 243 p.Enum = f[5:] 244 case f == "proto3": 245 p.proto3 = true 246 case f == "oneof": 247 p.oneof = true 248 case strings.HasPrefix(f, "def="): 249 p.HasDefault = true 250 p.Default = f[4:] // rest of string 251 if i+1 < len(fields) { 252 // Commas aren't escaped, and def is always last. 253 p.Default += "," + strings.Join(fields[i+1:], ",") 254 break outer 255 } 256 } 257 } 258} 259 260var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem() 261 262// setFieldProps initializes the field properties for submessages and maps. 263func (p *Properties) setFieldProps(typ reflect.Type, f *reflect.StructField, lockGetProp bool) { 264 switch t1 := typ; t1.Kind() { 265 case reflect.Ptr: 266 if t1.Elem().Kind() == reflect.Struct { 267 p.stype = t1.Elem() 268 } 269 270 case reflect.Slice: 271 if t2 := t1.Elem(); t2.Kind() == reflect.Ptr && t2.Elem().Kind() == reflect.Struct { 272 p.stype = t2.Elem() 273 } 274 275 case reflect.Map: 276 p.mtype = t1 277 p.MapKeyProp = &Properties{} 278 p.MapKeyProp.init(reflect.PtrTo(p.mtype.Key()), "Key", f.Tag.Get("protobuf_key"), nil, lockGetProp) 279 p.MapValProp = &Properties{} 280 vtype := p.mtype.Elem() 281 if vtype.Kind() != reflect.Ptr && vtype.Kind() != reflect.Slice { 282 // The value type is not a message (*T) or bytes ([]byte), 283 // so we need encoders for the pointer to this type. 284 vtype = reflect.PtrTo(vtype) 285 } 286 p.MapValProp.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp) 287 } 288 289 if p.stype != nil { 290 if lockGetProp { 291 p.sprop = GetProperties(p.stype) 292 } else { 293 p.sprop = getPropertiesLocked(p.stype) 294 } 295 } 296} 297 298var ( 299 marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() 300) 301 302// Init populates the properties from a protocol buffer struct tag. 303func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) { 304 p.init(typ, name, tag, f, true) 305} 306 307func (p *Properties) init(typ reflect.Type, name, tag string, f *reflect.StructField, lockGetProp bool) { 308 // "bytes,49,opt,def=hello!" 309 p.Name = name 310 p.OrigName = name 311 if tag == "" { 312 return 313 } 314 p.Parse(tag) 315 p.setFieldProps(typ, f, lockGetProp) 316} 317 318var ( 319 propertiesMu sync.RWMutex 320 propertiesMap = make(map[reflect.Type]*StructProperties) 321) 322 323// GetProperties returns the list of properties for the type represented by t. 324// t must represent a generated struct type of a protocol message. 325func GetProperties(t reflect.Type) *StructProperties { 326 if t.Kind() != reflect.Struct { 327 panic("proto: type must have kind struct") 328 } 329 330 // Most calls to GetProperties in a long-running program will be 331 // retrieving details for types we have seen before. 332 propertiesMu.RLock() 333 sprop, ok := propertiesMap[t] 334 propertiesMu.RUnlock() 335 if ok { 336 return sprop 337 } 338 339 propertiesMu.Lock() 340 sprop = getPropertiesLocked(t) 341 propertiesMu.Unlock() 342 return sprop 343} 344 345type ( 346 oneofFuncsIface interface { 347 XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{}) 348 } 349 oneofWrappersIface interface { 350 XXX_OneofWrappers() []interface{} 351 } 352) 353 354// getPropertiesLocked requires that propertiesMu is held. 355func getPropertiesLocked(t reflect.Type) *StructProperties { 356 if prop, ok := propertiesMap[t]; ok { 357 return prop 358 } 359 360 prop := new(StructProperties) 361 // in case of recursive protos, fill this in now. 362 propertiesMap[t] = prop 363 364 // build properties 365 prop.Prop = make([]*Properties, t.NumField()) 366 prop.order = make([]int, t.NumField()) 367 368 for i := 0; i < t.NumField(); i++ { 369 f := t.Field(i) 370 p := new(Properties) 371 name := f.Name 372 p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false) 373 374 oneof := f.Tag.Get("protobuf_oneof") // special case 375 if oneof != "" { 376 // Oneof fields don't use the traditional protobuf tag. 377 p.OrigName = oneof 378 } 379 prop.Prop[i] = p 380 prop.order[i] = i 381 if debug { 382 print(i, " ", f.Name, " ", t.String(), " ") 383 if p.Tag > 0 { 384 print(p.String()) 385 } 386 print("\n") 387 } 388 } 389 390 // Re-order prop.order. 391 sort.Sort(prop) 392 393 var oots []interface{} 394 switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) { 395 case oneofFuncsIface: 396 _, _, _, oots = m.XXX_OneofFuncs() 397 case oneofWrappersIface: 398 oots = m.XXX_OneofWrappers() 399 } 400 if len(oots) > 0 { 401 // Interpret oneof metadata. 402 prop.OneofTypes = make(map[string]*OneofProperties) 403 for _, oot := range oots { 404 oop := &OneofProperties{ 405 Type: reflect.ValueOf(oot).Type(), // *T 406 Prop: new(Properties), 407 } 408 sft := oop.Type.Elem().Field(0) 409 oop.Prop.Name = sft.Name 410 oop.Prop.Parse(sft.Tag.Get("protobuf")) 411 // There will be exactly one interface field that 412 // this new value is assignable to. 413 for i := 0; i < t.NumField(); i++ { 414 f := t.Field(i) 415 if f.Type.Kind() != reflect.Interface { 416 continue 417 } 418 if !oop.Type.AssignableTo(f.Type) { 419 continue 420 } 421 oop.Field = i 422 break 423 } 424 prop.OneofTypes[oop.Prop.OrigName] = oop 425 } 426 } 427 428 // build required counts 429 // build tags 430 reqCount := 0 431 prop.decoderOrigNames = make(map[string]int) 432 for i, p := range prop.Prop { 433 if strings.HasPrefix(p.Name, "XXX_") { 434 // Internal fields should not appear in tags/origNames maps. 435 // They are handled specially when encoding and decoding. 436 continue 437 } 438 if p.Required { 439 reqCount++ 440 } 441 prop.decoderTags.put(p.Tag, i) 442 prop.decoderOrigNames[p.OrigName] = i 443 } 444 prop.reqCount = reqCount 445 446 return prop 447} 448 449// A global registry of enum types. 450// The generated code will register the generated maps by calling RegisterEnum. 451 452var enumValueMaps = make(map[string]map[string]int32) 453 454// RegisterEnum is called from the generated code to install the enum descriptor 455// maps into the global table to aid parsing text format protocol buffers. 456func RegisterEnum(typeName string, unusedNameMap map[int32]string, valueMap map[string]int32) { 457 if _, ok := enumValueMaps[typeName]; ok { 458 panic("proto: duplicate enum registered: " + typeName) 459 } 460 enumValueMaps[typeName] = valueMap 461} 462 463// EnumValueMap returns the mapping from names to integers of the 464// enum type enumType, or a nil if not found. 465func EnumValueMap(enumType string) map[string]int32 { 466 return enumValueMaps[enumType] 467} 468 469// A registry of all linked message types. 470// The string is a fully-qualified proto name ("pkg.Message"). 471var ( 472 protoTypedNils = make(map[string]Message) // a map from proto names to typed nil pointers 473 protoMapTypes = make(map[string]reflect.Type) // a map from proto names to map types 474 revProtoTypes = make(map[reflect.Type]string) 475) 476 477// RegisterType is called from generated code and maps from the fully qualified 478// proto name to the type (pointer to struct) of the protocol buffer. 479func RegisterType(x Message, name string) { 480 if _, ok := protoTypedNils[name]; ok { 481 // TODO: Some day, make this a panic. 482 log.Printf("proto: duplicate proto type registered: %s", name) 483 return 484 } 485 t := reflect.TypeOf(x) 486 if v := reflect.ValueOf(x); v.Kind() == reflect.Ptr && v.Pointer() == 0 { 487 // Generated code always calls RegisterType with nil x. 488 // This check is just for extra safety. 489 protoTypedNils[name] = x 490 } else { 491 protoTypedNils[name] = reflect.Zero(t).Interface().(Message) 492 } 493 revProtoTypes[t] = name 494} 495 496// RegisterMapType is called from generated code and maps from the fully qualified 497// proto name to the native map type of the proto map definition. 498func RegisterMapType(x interface{}, name string) { 499 if reflect.TypeOf(x).Kind() != reflect.Map { 500 panic(fmt.Sprintf("RegisterMapType(%T, %q); want map", x, name)) 501 } 502 if _, ok := protoMapTypes[name]; ok { 503 log.Printf("proto: duplicate proto type registered: %s", name) 504 return 505 } 506 t := reflect.TypeOf(x) 507 protoMapTypes[name] = t 508 revProtoTypes[t] = name 509} 510 511// MessageName returns the fully-qualified proto name for the given message type. 512func MessageName(x Message) string { 513 type xname interface { 514 XXX_MessageName() string 515 } 516 if m, ok := x.(xname); ok { 517 return m.XXX_MessageName() 518 } 519 return revProtoTypes[reflect.TypeOf(x)] 520} 521 522// MessageType returns the message type (pointer to struct) for a named message. 523// The type is not guaranteed to implement proto.Message if the name refers to a 524// map entry. 525func MessageType(name string) reflect.Type { 526 if t, ok := protoTypedNils[name]; ok { 527 return reflect.TypeOf(t) 528 } 529 return protoMapTypes[name] 530} 531 532// A registry of all linked proto files. 533var ( 534 protoFiles = make(map[string][]byte) // file name => fileDescriptor 535) 536 537// RegisterFile is called from generated code and maps from the 538// full file name of a .proto file to its compressed FileDescriptorProto. 539func RegisterFile(filename string, fileDescriptor []byte) { 540 protoFiles[filename] = fileDescriptor 541} 542 543// FileDescriptor returns the compressed FileDescriptorProto for a .proto file. 544func FileDescriptor(filename string) []byte { return protoFiles[filename] } 545