1// Go support for Protocol Buffers - Google's data interchange format 2// 3// Copyright 2010 The Go Authors. All rights reserved. 4// https://github.com/golang/protobuf 5// 6// Redistribution and use in source and binary forms, with or without 7// modification, are permitted provided that the following conditions are 8// met: 9// 10// * Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// * Redistributions in binary form must reproduce the above 13// copyright notice, this list of conditions and the following disclaimer 14// in the documentation and/or other materials provided with the 15// distribution. 16// * Neither the name of Google Inc. nor the names of its 17// contributors may be used to endorse or promote products derived from 18// this software without specific prior written permission. 19// 20// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32package proto 33 34/* 35 * Routines for encoding data into the wire format for protocol buffers. 36 */ 37 38import ( 39 "fmt" 40 "log" 41 "os" 42 "reflect" 43 "sort" 44 "strconv" 45 "strings" 46 "sync" 47) 48 49const debug bool = false 50 51// Constants that identify the encoding of a value on the wire. 52const ( 53 WireVarint = 0 54 WireFixed64 = 1 55 WireBytes = 2 56 WireStartGroup = 3 57 WireEndGroup = 4 58 WireFixed32 = 5 59) 60 61// tagMap is an optimization over map[int]int for typical protocol buffer 62// use-cases. Encoded protocol buffers are often in tag order with small tag 63// numbers. 64type tagMap struct { 65 fastTags []int 66 slowTags map[int]int 67} 68 69// tagMapFastLimit is the upper bound on the tag number that will be stored in 70// the tagMap slice rather than its map. 71const tagMapFastLimit = 1024 72 73func (p *tagMap) get(t int) (int, bool) { 74 if t > 0 && t < tagMapFastLimit { 75 if t >= len(p.fastTags) { 76 return 0, false 77 } 78 fi := p.fastTags[t] 79 return fi, fi >= 0 80 } 81 fi, ok := p.slowTags[t] 82 return fi, ok 83} 84 85func (p *tagMap) put(t int, fi int) { 86 if t > 0 && t < tagMapFastLimit { 87 for len(p.fastTags) < t+1 { 88 p.fastTags = append(p.fastTags, -1) 89 } 90 p.fastTags[t] = fi 91 return 92 } 93 if p.slowTags == nil { 94 p.slowTags = make(map[int]int) 95 } 96 p.slowTags[t] = fi 97} 98 99// StructProperties represents properties for all the fields of a struct. 100// decoderTags and decoderOrigNames should only be used by the decoder. 101type StructProperties struct { 102 Prop []*Properties // properties for each field 103 reqCount int // required count 104 decoderTags tagMap // map from proto tag to struct field number 105 decoderOrigNames map[string]int // map from original name to struct field number 106 order []int // list of struct field numbers in tag order 107 108 // OneofTypes contains information about the oneof fields in this message. 109 // It is keyed by the original name of a field. 110 OneofTypes map[string]*OneofProperties 111} 112 113// OneofProperties represents information about a specific field in a oneof. 114type OneofProperties struct { 115 Type reflect.Type // pointer to generated struct type for this oneof field 116 Field int // struct field number of the containing oneof in the message 117 Prop *Properties 118} 119 120// Implement the sorting interface so we can sort the fields in tag order, as recommended by the spec. 121// See encode.go, (*Buffer).enc_struct. 122 123func (sp *StructProperties) Len() int { return len(sp.order) } 124func (sp *StructProperties) Less(i, j int) bool { 125 return sp.Prop[sp.order[i]].Tag < sp.Prop[sp.order[j]].Tag 126} 127func (sp *StructProperties) Swap(i, j int) { sp.order[i], sp.order[j] = sp.order[j], sp.order[i] } 128 129// Properties represents the protocol-specific behavior of a single struct field. 130type Properties struct { 131 Name string // name of the field, for error messages 132 OrigName string // original name before protocol compiler (always set) 133 JSONName string // name to use for JSON; determined by protoc 134 Wire string 135 WireType int 136 Tag int 137 Required bool 138 Optional bool 139 Repeated bool 140 Packed bool // relevant for repeated primitives only 141 Enum string // set for enum types only 142 proto3 bool // whether this is known to be a proto3 field 143 oneof bool // whether this is a oneof field 144 145 Default string // default value 146 HasDefault bool // whether an explicit default was provided 147 148 stype reflect.Type // set for struct types only 149 sprop *StructProperties // set for struct types only 150 151 mtype reflect.Type // set for map types only 152 MapKeyProp *Properties // set for map types only 153 MapValProp *Properties // set for map types only 154} 155 156// String formats the properties in the protobuf struct field tag style. 157func (p *Properties) String() string { 158 s := p.Wire 159 s += "," 160 s += strconv.Itoa(p.Tag) 161 if p.Required { 162 s += ",req" 163 } 164 if p.Optional { 165 s += ",opt" 166 } 167 if p.Repeated { 168 s += ",rep" 169 } 170 if p.Packed { 171 s += ",packed" 172 } 173 s += ",name=" + p.OrigName 174 if p.JSONName != p.OrigName { 175 s += ",json=" + p.JSONName 176 } 177 if p.proto3 { 178 s += ",proto3" 179 } 180 if p.oneof { 181 s += ",oneof" 182 } 183 if len(p.Enum) > 0 { 184 s += ",enum=" + p.Enum 185 } 186 if p.HasDefault { 187 s += ",def=" + p.Default 188 } 189 return s 190} 191 192// Parse populates p by parsing a string in the protobuf struct field tag style. 193func (p *Properties) Parse(s string) { 194 // "bytes,49,opt,name=foo,def=hello!" 195 fields := strings.Split(s, ",") // breaks def=, but handled below. 196 if len(fields) < 2 { 197 fmt.Fprintf(os.Stderr, "proto: tag has too few fields: %q\n", s) 198 return 199 } 200 201 p.Wire = fields[0] 202 switch p.Wire { 203 case "varint": 204 p.WireType = WireVarint 205 case "fixed32": 206 p.WireType = WireFixed32 207 case "fixed64": 208 p.WireType = WireFixed64 209 case "zigzag32": 210 p.WireType = WireVarint 211 case "zigzag64": 212 p.WireType = WireVarint 213 case "bytes", "group": 214 p.WireType = WireBytes 215 // no numeric converter for non-numeric types 216 default: 217 fmt.Fprintf(os.Stderr, "proto: tag has unknown wire type: %q\n", s) 218 return 219 } 220 221 var err error 222 p.Tag, err = strconv.Atoi(fields[1]) 223 if err != nil { 224 return 225 } 226 227outer: 228 for i := 2; i < len(fields); i++ { 229 f := fields[i] 230 switch { 231 case f == "req": 232 p.Required = true 233 case f == "opt": 234 p.Optional = true 235 case f == "rep": 236 p.Repeated = true 237 case f == "packed": 238 p.Packed = true 239 case strings.HasPrefix(f, "name="): 240 p.OrigName = f[5:] 241 case strings.HasPrefix(f, "json="): 242 p.JSONName = f[5:] 243 case strings.HasPrefix(f, "enum="): 244 p.Enum = f[5:] 245 case f == "proto3": 246 p.proto3 = true 247 case f == "oneof": 248 p.oneof = true 249 case strings.HasPrefix(f, "def="): 250 p.HasDefault = true 251 p.Default = f[4:] // rest of string 252 if i+1 < len(fields) { 253 // Commas aren't escaped, and def is always last. 254 p.Default += "," + strings.Join(fields[i+1:], ",") 255 break outer 256 } 257 } 258 } 259} 260 261var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem() 262 263// setFieldProps initializes the field properties for submessages and maps. 264func (p *Properties) setFieldProps(typ reflect.Type, f *reflect.StructField, lockGetProp bool) { 265 switch t1 := typ; t1.Kind() { 266 case reflect.Ptr: 267 if t1.Elem().Kind() == reflect.Struct { 268 p.stype = t1.Elem() 269 } 270 271 case reflect.Slice: 272 if t2 := t1.Elem(); t2.Kind() == reflect.Ptr && t2.Elem().Kind() == reflect.Struct { 273 p.stype = t2.Elem() 274 } 275 276 case reflect.Map: 277 p.mtype = t1 278 p.MapKeyProp = &Properties{} 279 p.MapKeyProp.init(reflect.PtrTo(p.mtype.Key()), "Key", f.Tag.Get("protobuf_key"), nil, lockGetProp) 280 p.MapValProp = &Properties{} 281 vtype := p.mtype.Elem() 282 if vtype.Kind() != reflect.Ptr && vtype.Kind() != reflect.Slice { 283 // The value type is not a message (*T) or bytes ([]byte), 284 // so we need encoders for the pointer to this type. 285 vtype = reflect.PtrTo(vtype) 286 } 287 p.MapValProp.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp) 288 } 289 290 if p.stype != nil { 291 if lockGetProp { 292 p.sprop = GetProperties(p.stype) 293 } else { 294 p.sprop = getPropertiesLocked(p.stype) 295 } 296 } 297} 298 299var ( 300 marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() 301) 302 303// Init populates the properties from a protocol buffer struct tag. 304func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) { 305 p.init(typ, name, tag, f, true) 306} 307 308func (p *Properties) init(typ reflect.Type, name, tag string, f *reflect.StructField, lockGetProp bool) { 309 // "bytes,49,opt,def=hello!" 310 p.Name = name 311 p.OrigName = name 312 if tag == "" { 313 return 314 } 315 p.Parse(tag) 316 p.setFieldProps(typ, f, lockGetProp) 317} 318 319var ( 320 propertiesMu sync.RWMutex 321 propertiesMap = make(map[reflect.Type]*StructProperties) 322) 323 324// GetProperties returns the list of properties for the type represented by t. 325// t must represent a generated struct type of a protocol message. 326func GetProperties(t reflect.Type) *StructProperties { 327 if t.Kind() != reflect.Struct { 328 panic("proto: type must have kind struct") 329 } 330 331 // Most calls to GetProperties in a long-running program will be 332 // retrieving details for types we have seen before. 333 propertiesMu.RLock() 334 sprop, ok := propertiesMap[t] 335 propertiesMu.RUnlock() 336 if ok { 337 return sprop 338 } 339 340 propertiesMu.Lock() 341 sprop = getPropertiesLocked(t) 342 propertiesMu.Unlock() 343 return sprop 344} 345 346type ( 347 oneofFuncsIface interface { 348 XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{}) 349 } 350 oneofWrappersIface interface { 351 XXX_OneofWrappers() []interface{} 352 } 353) 354 355// getPropertiesLocked requires that propertiesMu is held. 356func getPropertiesLocked(t reflect.Type) *StructProperties { 357 if prop, ok := propertiesMap[t]; ok { 358 return prop 359 } 360 361 prop := new(StructProperties) 362 // in case of recursive protos, fill this in now. 363 propertiesMap[t] = prop 364 365 // build properties 366 prop.Prop = make([]*Properties, t.NumField()) 367 prop.order = make([]int, t.NumField()) 368 369 for i := 0; i < t.NumField(); i++ { 370 f := t.Field(i) 371 p := new(Properties) 372 name := f.Name 373 p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false) 374 375 oneof := f.Tag.Get("protobuf_oneof") // special case 376 if oneof != "" { 377 // Oneof fields don't use the traditional protobuf tag. 378 p.OrigName = oneof 379 } 380 prop.Prop[i] = p 381 prop.order[i] = i 382 if debug { 383 print(i, " ", f.Name, " ", t.String(), " ") 384 if p.Tag > 0 { 385 print(p.String()) 386 } 387 print("\n") 388 } 389 } 390 391 // Re-order prop.order. 392 sort.Sort(prop) 393 394 var oots []interface{} 395 switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) { 396 case oneofFuncsIface: 397 _, _, _, oots = m.XXX_OneofFuncs() 398 case oneofWrappersIface: 399 oots = m.XXX_OneofWrappers() 400 } 401 if len(oots) > 0 { 402 // Interpret oneof metadata. 403 prop.OneofTypes = make(map[string]*OneofProperties) 404 for _, oot := range oots { 405 oop := &OneofProperties{ 406 Type: reflect.ValueOf(oot).Type(), // *T 407 Prop: new(Properties), 408 } 409 sft := oop.Type.Elem().Field(0) 410 oop.Prop.Name = sft.Name 411 oop.Prop.Parse(sft.Tag.Get("protobuf")) 412 // There will be exactly one interface field that 413 // this new value is assignable to. 414 for i := 0; i < t.NumField(); i++ { 415 f := t.Field(i) 416 if f.Type.Kind() != reflect.Interface { 417 continue 418 } 419 if !oop.Type.AssignableTo(f.Type) { 420 continue 421 } 422 oop.Field = i 423 break 424 } 425 prop.OneofTypes[oop.Prop.OrigName] = oop 426 } 427 } 428 429 // build required counts 430 // build tags 431 reqCount := 0 432 prop.decoderOrigNames = make(map[string]int) 433 for i, p := range prop.Prop { 434 if strings.HasPrefix(p.Name, "XXX_") { 435 // Internal fields should not appear in tags/origNames maps. 436 // They are handled specially when encoding and decoding. 437 continue 438 } 439 if p.Required { 440 reqCount++ 441 } 442 prop.decoderTags.put(p.Tag, i) 443 prop.decoderOrigNames[p.OrigName] = i 444 } 445 prop.reqCount = reqCount 446 447 return prop 448} 449 450// A global registry of enum types. 451// The generated code will register the generated maps by calling RegisterEnum. 452 453var enumValueMaps = make(map[string]map[string]int32) 454 455// RegisterEnum is called from the generated code to install the enum descriptor 456// maps into the global table to aid parsing text format protocol buffers. 457func RegisterEnum(typeName string, unusedNameMap map[int32]string, valueMap map[string]int32) { 458 if _, ok := enumValueMaps[typeName]; ok { 459 panic("proto: duplicate enum registered: " + typeName) 460 } 461 enumValueMaps[typeName] = valueMap 462} 463 464// EnumValueMap returns the mapping from names to integers of the 465// enum type enumType, or a nil if not found. 466func EnumValueMap(enumType string) map[string]int32 { 467 return enumValueMaps[enumType] 468} 469 470// A registry of all linked message types. 471// The string is a fully-qualified proto name ("pkg.Message"). 472var ( 473 protoTypedNils = make(map[string]Message) // a map from proto names to typed nil pointers 474 protoMapTypes = make(map[string]reflect.Type) // a map from proto names to map types 475 revProtoTypes = make(map[reflect.Type]string) 476) 477 478// RegisterType is called from generated code and maps from the fully qualified 479// proto name to the type (pointer to struct) of the protocol buffer. 480func RegisterType(x Message, name string) { 481 if _, ok := protoTypedNils[name]; ok { 482 // TODO: Some day, make this a panic. 483 log.Printf("proto: duplicate proto type registered: %s", name) 484 return 485 } 486 t := reflect.TypeOf(x) 487 if v := reflect.ValueOf(x); v.Kind() == reflect.Ptr && v.Pointer() == 0 { 488 // Generated code always calls RegisterType with nil x. 489 // This check is just for extra safety. 490 protoTypedNils[name] = x 491 } else { 492 protoTypedNils[name] = reflect.Zero(t).Interface().(Message) 493 } 494 revProtoTypes[t] = name 495} 496 497// RegisterMapType is called from generated code and maps from the fully qualified 498// proto name to the native map type of the proto map definition. 499func RegisterMapType(x interface{}, name string) { 500 if reflect.TypeOf(x).Kind() != reflect.Map { 501 panic(fmt.Sprintf("RegisterMapType(%T, %q); want map", x, name)) 502 } 503 if _, ok := protoMapTypes[name]; ok { 504 log.Printf("proto: duplicate proto type registered: %s", name) 505 return 506 } 507 t := reflect.TypeOf(x) 508 protoMapTypes[name] = t 509 revProtoTypes[t] = name 510} 511 512// MessageName returns the fully-qualified proto name for the given message type. 513func MessageName(x Message) string { 514 type xname interface { 515 XXX_MessageName() string 516 } 517 if m, ok := x.(xname); ok { 518 return m.XXX_MessageName() 519 } 520 return revProtoTypes[reflect.TypeOf(x)] 521} 522 523// MessageType returns the message type (pointer to struct) for a named message. 524// The type is not guaranteed to implement proto.Message if the name refers to a 525// map entry. 526func MessageType(name string) reflect.Type { 527 if t, ok := protoTypedNils[name]; ok { 528 return reflect.TypeOf(t) 529 } 530 return protoMapTypes[name] 531} 532 533// A registry of all linked proto files. 534var ( 535 protoFiles = make(map[string][]byte) // file name => fileDescriptor 536) 537 538// RegisterFile is called from generated code and maps from the 539// full file name of a .proto file to its compressed FileDescriptorProto. 540func RegisterFile(filename string, fileDescriptor []byte) { 541 protoFiles[filename] = fileDescriptor 542} 543 544// FileDescriptor returns the compressed FileDescriptorProto for a .proto file. 545func FileDescriptor(filename string) []byte { return protoFiles[filename] } 546