1// Go support for Protocol Buffers - Google's data interchange format 2// 3// Copyright 2015 The Go Authors. All rights reserved. 4// https://github.com/golang/protobuf 5// 6// Redistribution and use in source and binary forms, with or without 7// modification, are permitted provided that the following conditions are 8// met: 9// 10// * Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// * Redistributions in binary form must reproduce the above 13// copyright notice, this list of conditions and the following disclaimer 14// in the documentation and/or other materials provided with the 15// distribution. 16// * Neither the name of Google Inc. nor the names of its 17// contributors may be used to endorse or promote products derived from 18// this software without specific prior written permission. 19// 20// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32/* 33Package jsonpb provides marshaling and unmarshaling between protocol buffers and JSON. 34It follows the specification at https://developers.google.com/protocol-buffers/docs/proto3#json. 35 36This package produces a different output than the standard "encoding/json" package, 37which does not operate correctly on protocol buffers. 38*/ 39package jsonpb 40 41import ( 42 "bytes" 43 "encoding/json" 44 "errors" 45 "fmt" 46 "io" 47 "math" 48 "reflect" 49 "sort" 50 "strconv" 51 "strings" 52 "time" 53 54 "github.com/golang/protobuf/proto" 55 56 stpb "github.com/golang/protobuf/ptypes/struct" 57) 58 59const secondInNanos = int64(time.Second / time.Nanosecond) 60const maxSecondsInDuration = 315576000000 61 62// Marshaler is a configurable object for converting between 63// protocol buffer objects and a JSON representation for them. 64type Marshaler struct { 65 // Whether to render enum values as integers, as opposed to string values. 66 EnumsAsInts bool 67 68 // Whether to render fields with zero values. 69 EmitDefaults bool 70 71 // A string to indent each level by. The presence of this field will 72 // also cause a space to appear between the field separator and 73 // value, and for newlines to be appear between fields and array 74 // elements. 75 Indent string 76 77 // Whether to use the original (.proto) name for fields. 78 OrigName bool 79 80 // A custom URL resolver to use when marshaling Any messages to JSON. 81 // If unset, the default resolution strategy is to extract the 82 // fully-qualified type name from the type URL and pass that to 83 // proto.MessageType(string). 84 AnyResolver AnyResolver 85} 86 87// AnyResolver takes a type URL, present in an Any message, and resolves it into 88// an instance of the associated message. 89type AnyResolver interface { 90 Resolve(typeUrl string) (proto.Message, error) 91} 92 93func defaultResolveAny(typeUrl string) (proto.Message, error) { 94 // Only the part of typeUrl after the last slash is relevant. 95 mname := typeUrl 96 if slash := strings.LastIndex(mname, "/"); slash >= 0 { 97 mname = mname[slash+1:] 98 } 99 mt := proto.MessageType(mname) 100 if mt == nil { 101 return nil, fmt.Errorf("unknown message type %q", mname) 102 } 103 return reflect.New(mt.Elem()).Interface().(proto.Message), nil 104} 105 106// JSONPBMarshaler is implemented by protobuf messages that customize the 107// way they are marshaled to JSON. Messages that implement this should 108// also implement JSONPBUnmarshaler so that the custom format can be 109// parsed. 110// 111// The JSON marshaling must follow the proto to JSON specification: 112// https://developers.google.com/protocol-buffers/docs/proto3#json 113type JSONPBMarshaler interface { 114 MarshalJSONPB(*Marshaler) ([]byte, error) 115} 116 117// JSONPBUnmarshaler is implemented by protobuf messages that customize 118// the way they are unmarshaled from JSON. Messages that implement this 119// should also implement JSONPBMarshaler so that the custom format can be 120// produced. 121// 122// The JSON unmarshaling must follow the JSON to proto specification: 123// https://developers.google.com/protocol-buffers/docs/proto3#json 124type JSONPBUnmarshaler interface { 125 UnmarshalJSONPB(*Unmarshaler, []byte) error 126} 127 128// Marshal marshals a protocol buffer into JSON. 129func (m *Marshaler) Marshal(out io.Writer, pb proto.Message) error { 130 v := reflect.ValueOf(pb) 131 if pb == nil || (v.Kind() == reflect.Ptr && v.IsNil()) { 132 return errors.New("Marshal called with nil") 133 } 134 // Check for unset required fields first. 135 if err := checkRequiredFields(pb); err != nil { 136 return err 137 } 138 writer := &errWriter{writer: out} 139 return m.marshalObject(writer, pb, "", "") 140} 141 142// MarshalToString converts a protocol buffer object to JSON string. 143func (m *Marshaler) MarshalToString(pb proto.Message) (string, error) { 144 var buf bytes.Buffer 145 if err := m.Marshal(&buf, pb); err != nil { 146 return "", err 147 } 148 return buf.String(), nil 149} 150 151type int32Slice []int32 152 153var nonFinite = map[string]float64{ 154 `"NaN"`: math.NaN(), 155 `"Infinity"`: math.Inf(1), 156 `"-Infinity"`: math.Inf(-1), 157} 158 159// For sorting extensions ids to ensure stable output. 160func (s int32Slice) Len() int { return len(s) } 161func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] } 162func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 163 164type wkt interface { 165 XXX_WellKnownType() string 166} 167 168var ( 169 wktType = reflect.TypeOf((*wkt)(nil)).Elem() 170 messageType = reflect.TypeOf((*proto.Message)(nil)).Elem() 171) 172 173// marshalObject writes a struct to the Writer. 174func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error { 175 if jsm, ok := v.(JSONPBMarshaler); ok { 176 b, err := jsm.MarshalJSONPB(m) 177 if err != nil { 178 return err 179 } 180 if typeURL != "" { 181 // we are marshaling this object to an Any type 182 var js map[string]*json.RawMessage 183 if err = json.Unmarshal(b, &js); err != nil { 184 return fmt.Errorf("type %T produced invalid JSON: %v", v, err) 185 } 186 turl, err := json.Marshal(typeURL) 187 if err != nil { 188 return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err) 189 } 190 js["@type"] = (*json.RawMessage)(&turl) 191 if m.Indent != "" { 192 b, err = json.MarshalIndent(js, indent, m.Indent) 193 } else { 194 b, err = json.Marshal(js) 195 } 196 if err != nil { 197 return err 198 } 199 } 200 201 out.write(string(b)) 202 return out.err 203 } 204 205 s := reflect.ValueOf(v).Elem() 206 207 // Handle well-known types. 208 if wkt, ok := v.(wkt); ok { 209 switch wkt.XXX_WellKnownType() { 210 case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value", 211 "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue": 212 // "Wrappers use the same representation in JSON 213 // as the wrapped primitive type, ..." 214 sprop := proto.GetProperties(s.Type()) 215 return m.marshalValue(out, sprop.Prop[0], s.Field(0), indent) 216 case "Any": 217 // Any is a bit more involved. 218 return m.marshalAny(out, v, indent) 219 case "Duration": 220 s, ns := s.Field(0).Int(), s.Field(1).Int() 221 if s < -maxSecondsInDuration || s > maxSecondsInDuration { 222 return fmt.Errorf("seconds out of range %v", s) 223 } 224 if ns <= -secondInNanos || ns >= secondInNanos { 225 return fmt.Errorf("ns out of range (%v, %v)", -secondInNanos, secondInNanos) 226 } 227 if (s > 0 && ns < 0) || (s < 0 && ns > 0) { 228 return errors.New("signs of seconds and nanos do not match") 229 } 230 // Generated output always contains 0, 3, 6, or 9 fractional digits, 231 // depending on required precision, followed by the suffix "s". 232 f := "%d.%09d" 233 if ns < 0 { 234 ns = -ns 235 if s == 0 { 236 f = "-%d.%09d" 237 } 238 } 239 x := fmt.Sprintf(f, s, ns) 240 x = strings.TrimSuffix(x, "000") 241 x = strings.TrimSuffix(x, "000") 242 x = strings.TrimSuffix(x, ".000") 243 out.write(`"`) 244 out.write(x) 245 out.write(`s"`) 246 return out.err 247 case "Struct", "ListValue": 248 // Let marshalValue handle the `Struct.fields` map or the `ListValue.values` slice. 249 // TODO: pass the correct Properties if needed. 250 return m.marshalValue(out, &proto.Properties{}, s.Field(0), indent) 251 case "Timestamp": 252 // "RFC 3339, where generated output will always be Z-normalized 253 // and uses 0, 3, 6 or 9 fractional digits." 254 s, ns := s.Field(0).Int(), s.Field(1).Int() 255 if ns < 0 || ns >= secondInNanos { 256 return fmt.Errorf("ns out of range [0, %v)", secondInNanos) 257 } 258 t := time.Unix(s, ns).UTC() 259 // time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits). 260 x := t.Format("2006-01-02T15:04:05.000000000") 261 x = strings.TrimSuffix(x, "000") 262 x = strings.TrimSuffix(x, "000") 263 x = strings.TrimSuffix(x, ".000") 264 out.write(`"`) 265 out.write(x) 266 out.write(`Z"`) 267 return out.err 268 case "Value": 269 // Value has a single oneof. 270 kind := s.Field(0) 271 if kind.IsNil() { 272 // "absence of any variant indicates an error" 273 return errors.New("nil Value") 274 } 275 // oneof -> *T -> T -> T.F 276 x := kind.Elem().Elem().Field(0) 277 // TODO: pass the correct Properties if needed. 278 return m.marshalValue(out, &proto.Properties{}, x, indent) 279 } 280 } 281 282 out.write("{") 283 if m.Indent != "" { 284 out.write("\n") 285 } 286 287 firstField := true 288 289 if typeURL != "" { 290 if err := m.marshalTypeURL(out, indent, typeURL); err != nil { 291 return err 292 } 293 firstField = false 294 } 295 296 for i := 0; i < s.NumField(); i++ { 297 value := s.Field(i) 298 valueField := s.Type().Field(i) 299 if strings.HasPrefix(valueField.Name, "XXX_") { 300 continue 301 } 302 303 // IsNil will panic on most value kinds. 304 switch value.Kind() { 305 case reflect.Chan, reflect.Func, reflect.Interface: 306 if value.IsNil() { 307 continue 308 } 309 } 310 311 if !m.EmitDefaults { 312 switch value.Kind() { 313 case reflect.Bool: 314 if !value.Bool() { 315 continue 316 } 317 case reflect.Int32, reflect.Int64: 318 if value.Int() == 0 { 319 continue 320 } 321 case reflect.Uint32, reflect.Uint64: 322 if value.Uint() == 0 { 323 continue 324 } 325 case reflect.Float32, reflect.Float64: 326 if value.Float() == 0 { 327 continue 328 } 329 case reflect.String: 330 if value.Len() == 0 { 331 continue 332 } 333 case reflect.Map, reflect.Ptr, reflect.Slice: 334 if value.IsNil() { 335 continue 336 } 337 } 338 } 339 340 // Oneof fields need special handling. 341 if valueField.Tag.Get("protobuf_oneof") != "" { 342 // value is an interface containing &T{real_value}. 343 sv := value.Elem().Elem() // interface -> *T -> T 344 value = sv.Field(0) 345 valueField = sv.Type().Field(0) 346 } 347 prop := jsonProperties(valueField, m.OrigName) 348 if !firstField { 349 m.writeSep(out) 350 } 351 if err := m.marshalField(out, prop, value, indent); err != nil { 352 return err 353 } 354 firstField = false 355 } 356 357 // Handle proto2 extensions. 358 if ep, ok := v.(proto.Message); ok { 359 extensions := proto.RegisteredExtensions(v) 360 // Sort extensions for stable output. 361 ids := make([]int32, 0, len(extensions)) 362 for id, desc := range extensions { 363 if !proto.HasExtension(ep, desc) { 364 continue 365 } 366 ids = append(ids, id) 367 } 368 sort.Sort(int32Slice(ids)) 369 for _, id := range ids { 370 desc := extensions[id] 371 if desc == nil { 372 // unknown extension 373 continue 374 } 375 ext, extErr := proto.GetExtension(ep, desc) 376 if extErr != nil { 377 return extErr 378 } 379 value := reflect.ValueOf(ext) 380 var prop proto.Properties 381 prop.Parse(desc.Tag) 382 prop.JSONName = fmt.Sprintf("[%s]", desc.Name) 383 if !firstField { 384 m.writeSep(out) 385 } 386 if err := m.marshalField(out, &prop, value, indent); err != nil { 387 return err 388 } 389 firstField = false 390 } 391 392 } 393 394 if m.Indent != "" { 395 out.write("\n") 396 out.write(indent) 397 } 398 out.write("}") 399 return out.err 400} 401 402func (m *Marshaler) writeSep(out *errWriter) { 403 if m.Indent != "" { 404 out.write(",\n") 405 } else { 406 out.write(",") 407 } 408} 409 410func (m *Marshaler) marshalAny(out *errWriter, any proto.Message, indent string) error { 411 // "If the Any contains a value that has a special JSON mapping, 412 // it will be converted as follows: {"@type": xxx, "value": yyy}. 413 // Otherwise, the value will be converted into a JSON object, 414 // and the "@type" field will be inserted to indicate the actual data type." 415 v := reflect.ValueOf(any).Elem() 416 turl := v.Field(0).String() 417 val := v.Field(1).Bytes() 418 419 var msg proto.Message 420 var err error 421 if m.AnyResolver != nil { 422 msg, err = m.AnyResolver.Resolve(turl) 423 } else { 424 msg, err = defaultResolveAny(turl) 425 } 426 if err != nil { 427 return err 428 } 429 430 if err := proto.Unmarshal(val, msg); err != nil { 431 return err 432 } 433 434 if _, ok := msg.(wkt); ok { 435 out.write("{") 436 if m.Indent != "" { 437 out.write("\n") 438 } 439 if err := m.marshalTypeURL(out, indent, turl); err != nil { 440 return err 441 } 442 m.writeSep(out) 443 if m.Indent != "" { 444 out.write(indent) 445 out.write(m.Indent) 446 out.write(`"value": `) 447 } else { 448 out.write(`"value":`) 449 } 450 if err := m.marshalObject(out, msg, indent+m.Indent, ""); err != nil { 451 return err 452 } 453 if m.Indent != "" { 454 out.write("\n") 455 out.write(indent) 456 } 457 out.write("}") 458 return out.err 459 } 460 461 return m.marshalObject(out, msg, indent, turl) 462} 463 464func (m *Marshaler) marshalTypeURL(out *errWriter, indent, typeURL string) error { 465 if m.Indent != "" { 466 out.write(indent) 467 out.write(m.Indent) 468 } 469 out.write(`"@type":`) 470 if m.Indent != "" { 471 out.write(" ") 472 } 473 b, err := json.Marshal(typeURL) 474 if err != nil { 475 return err 476 } 477 out.write(string(b)) 478 return out.err 479} 480 481// marshalField writes field description and value to the Writer. 482func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error { 483 if m.Indent != "" { 484 out.write(indent) 485 out.write(m.Indent) 486 } 487 out.write(`"`) 488 out.write(prop.JSONName) 489 out.write(`":`) 490 if m.Indent != "" { 491 out.write(" ") 492 } 493 if err := m.marshalValue(out, prop, v, indent); err != nil { 494 return err 495 } 496 return nil 497} 498 499// marshalValue writes the value to the Writer. 500func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error { 501 var err error 502 v = reflect.Indirect(v) 503 504 // Handle nil pointer 505 if v.Kind() == reflect.Invalid { 506 out.write("null") 507 return out.err 508 } 509 510 // Handle repeated elements. 511 if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 { 512 out.write("[") 513 comma := "" 514 for i := 0; i < v.Len(); i++ { 515 sliceVal := v.Index(i) 516 out.write(comma) 517 if m.Indent != "" { 518 out.write("\n") 519 out.write(indent) 520 out.write(m.Indent) 521 out.write(m.Indent) 522 } 523 if err := m.marshalValue(out, prop, sliceVal, indent+m.Indent); err != nil { 524 return err 525 } 526 comma = "," 527 } 528 if m.Indent != "" { 529 out.write("\n") 530 out.write(indent) 531 out.write(m.Indent) 532 } 533 out.write("]") 534 return out.err 535 } 536 537 // Handle well-known types. 538 // Most are handled up in marshalObject (because 99% are messages). 539 if v.Type().Implements(wktType) { 540 wkt := v.Interface().(wkt) 541 switch wkt.XXX_WellKnownType() { 542 case "NullValue": 543 out.write("null") 544 return out.err 545 } 546 } 547 548 // Handle enumerations. 549 if !m.EnumsAsInts && prop.Enum != "" { 550 // Unknown enum values will are stringified by the proto library as their 551 // value. Such values should _not_ be quoted or they will be interpreted 552 // as an enum string instead of their value. 553 enumStr := v.Interface().(fmt.Stringer).String() 554 var valStr string 555 if v.Kind() == reflect.Ptr { 556 valStr = strconv.Itoa(int(v.Elem().Int())) 557 } else { 558 valStr = strconv.Itoa(int(v.Int())) 559 } 560 isKnownEnum := enumStr != valStr 561 if isKnownEnum { 562 out.write(`"`) 563 } 564 out.write(enumStr) 565 if isKnownEnum { 566 out.write(`"`) 567 } 568 return out.err 569 } 570 571 // Handle nested messages. 572 if v.Kind() == reflect.Struct { 573 return m.marshalObject(out, v.Addr().Interface().(proto.Message), indent+m.Indent, "") 574 } 575 576 // Handle maps. 577 // Since Go randomizes map iteration, we sort keys for stable output. 578 if v.Kind() == reflect.Map { 579 out.write(`{`) 580 keys := v.MapKeys() 581 sort.Sort(mapKeys(keys)) 582 for i, k := range keys { 583 if i > 0 { 584 out.write(`,`) 585 } 586 if m.Indent != "" { 587 out.write("\n") 588 out.write(indent) 589 out.write(m.Indent) 590 out.write(m.Indent) 591 } 592 593 // TODO handle map key prop properly 594 b, err := json.Marshal(k.Interface()) 595 if err != nil { 596 return err 597 } 598 s := string(b) 599 600 // If the JSON is not a string value, encode it again to make it one. 601 if !strings.HasPrefix(s, `"`) { 602 b, err := json.Marshal(s) 603 if err != nil { 604 return err 605 } 606 s = string(b) 607 } 608 609 out.write(s) 610 out.write(`:`) 611 if m.Indent != "" { 612 out.write(` `) 613 } 614 615 vprop := prop 616 if prop != nil && prop.MapValProp != nil { 617 vprop = prop.MapValProp 618 } 619 if err := m.marshalValue(out, vprop, v.MapIndex(k), indent+m.Indent); err != nil { 620 return err 621 } 622 } 623 if m.Indent != "" { 624 out.write("\n") 625 out.write(indent) 626 out.write(m.Indent) 627 } 628 out.write(`}`) 629 return out.err 630 } 631 632 // Handle non-finite floats, e.g. NaN, Infinity and -Infinity. 633 if v.Kind() == reflect.Float32 || v.Kind() == reflect.Float64 { 634 f := v.Float() 635 var sval string 636 switch { 637 case math.IsInf(f, 1): 638 sval = `"Infinity"` 639 case math.IsInf(f, -1): 640 sval = `"-Infinity"` 641 case math.IsNaN(f): 642 sval = `"NaN"` 643 } 644 if sval != "" { 645 out.write(sval) 646 return out.err 647 } 648 } 649 650 // Default handling defers to the encoding/json library. 651 b, err := json.Marshal(v.Interface()) 652 if err != nil { 653 return err 654 } 655 needToQuote := string(b[0]) != `"` && (v.Kind() == reflect.Int64 || v.Kind() == reflect.Uint64) 656 if needToQuote { 657 out.write(`"`) 658 } 659 out.write(string(b)) 660 if needToQuote { 661 out.write(`"`) 662 } 663 return out.err 664} 665 666// Unmarshaler is a configurable object for converting from a JSON 667// representation to a protocol buffer object. 668type Unmarshaler struct { 669 // Whether to allow messages to contain unknown fields, as opposed to 670 // failing to unmarshal. 671 AllowUnknownFields bool 672 673 // A custom URL resolver to use when unmarshaling Any messages from JSON. 674 // If unset, the default resolution strategy is to extract the 675 // fully-qualified type name from the type URL and pass that to 676 // proto.MessageType(string). 677 AnyResolver AnyResolver 678} 679 680// UnmarshalNext unmarshals the next protocol buffer from a JSON object stream. 681// This function is lenient and will decode any options permutations of the 682// related Marshaler. 683func (u *Unmarshaler) UnmarshalNext(dec *json.Decoder, pb proto.Message) error { 684 inputValue := json.RawMessage{} 685 if err := dec.Decode(&inputValue); err != nil { 686 return err 687 } 688 if err := u.unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil); err != nil { 689 return err 690 } 691 return checkRequiredFields(pb) 692} 693 694// Unmarshal unmarshals a JSON object stream into a protocol 695// buffer. This function is lenient and will decode any options 696// permutations of the related Marshaler. 697func (u *Unmarshaler) Unmarshal(r io.Reader, pb proto.Message) error { 698 dec := json.NewDecoder(r) 699 return u.UnmarshalNext(dec, pb) 700} 701 702// UnmarshalNext unmarshals the next protocol buffer from a JSON object stream. 703// This function is lenient and will decode any options permutations of the 704// related Marshaler. 705func UnmarshalNext(dec *json.Decoder, pb proto.Message) error { 706 return new(Unmarshaler).UnmarshalNext(dec, pb) 707} 708 709// Unmarshal unmarshals a JSON object stream into a protocol 710// buffer. This function is lenient and will decode any options 711// permutations of the related Marshaler. 712func Unmarshal(r io.Reader, pb proto.Message) error { 713 return new(Unmarshaler).Unmarshal(r, pb) 714} 715 716// UnmarshalString will populate the fields of a protocol buffer based 717// on a JSON string. This function is lenient and will decode any options 718// permutations of the related Marshaler. 719func UnmarshalString(str string, pb proto.Message) error { 720 return new(Unmarshaler).Unmarshal(strings.NewReader(str), pb) 721} 722 723// unmarshalValue converts/copies a value into the target. 724// prop may be nil. 725func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error { 726 targetType := target.Type() 727 728 // Allocate memory for pointer fields. 729 if targetType.Kind() == reflect.Ptr { 730 // If input value is "null" and target is a pointer type, then the field should be treated as not set 731 // UNLESS the target is structpb.Value, in which case it should be set to structpb.NullValue. 732 _, isJSONPBUnmarshaler := target.Interface().(JSONPBUnmarshaler) 733 if string(inputValue) == "null" && targetType != reflect.TypeOf(&stpb.Value{}) && !isJSONPBUnmarshaler { 734 return nil 735 } 736 target.Set(reflect.New(targetType.Elem())) 737 738 return u.unmarshalValue(target.Elem(), inputValue, prop) 739 } 740 741 if jsu, ok := target.Addr().Interface().(JSONPBUnmarshaler); ok { 742 return jsu.UnmarshalJSONPB(u, []byte(inputValue)) 743 } 744 745 // Handle well-known types that are not pointers. 746 if w, ok := target.Addr().Interface().(wkt); ok { 747 switch w.XXX_WellKnownType() { 748 case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value", 749 "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue": 750 return u.unmarshalValue(target.Field(0), inputValue, prop) 751 case "Any": 752 // Use json.RawMessage pointer type instead of value to support pre-1.8 version. 753 // 1.8 changed RawMessage.MarshalJSON from pointer type to value type, see 754 // https://github.com/golang/go/issues/14493 755 var jsonFields map[string]*json.RawMessage 756 if err := json.Unmarshal(inputValue, &jsonFields); err != nil { 757 return err 758 } 759 760 val, ok := jsonFields["@type"] 761 if !ok || val == nil { 762 return errors.New("Any JSON doesn't have '@type'") 763 } 764 765 var turl string 766 if err := json.Unmarshal([]byte(*val), &turl); err != nil { 767 return fmt.Errorf("can't unmarshal Any's '@type': %q", *val) 768 } 769 target.Field(0).SetString(turl) 770 771 var m proto.Message 772 var err error 773 if u.AnyResolver != nil { 774 m, err = u.AnyResolver.Resolve(turl) 775 } else { 776 m, err = defaultResolveAny(turl) 777 } 778 if err != nil { 779 return err 780 } 781 782 if _, ok := m.(wkt); ok { 783 val, ok := jsonFields["value"] 784 if !ok { 785 return errors.New("Any JSON doesn't have 'value'") 786 } 787 788 if err := u.unmarshalValue(reflect.ValueOf(m).Elem(), *val, nil); err != nil { 789 return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err) 790 } 791 } else { 792 delete(jsonFields, "@type") 793 nestedProto, err := json.Marshal(jsonFields) 794 if err != nil { 795 return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err) 796 } 797 798 if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), nestedProto, nil); err != nil { 799 return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err) 800 } 801 } 802 803 b, err := proto.Marshal(m) 804 if err != nil { 805 return fmt.Errorf("can't marshal proto %T into Any.Value: %v", m, err) 806 } 807 target.Field(1).SetBytes(b) 808 809 return nil 810 case "Duration": 811 unq, err := unquote(string(inputValue)) 812 if err != nil { 813 return err 814 } 815 816 d, err := time.ParseDuration(unq) 817 if err != nil { 818 return fmt.Errorf("bad Duration: %v", err) 819 } 820 821 ns := d.Nanoseconds() 822 s := ns / 1e9 823 ns %= 1e9 824 target.Field(0).SetInt(s) 825 target.Field(1).SetInt(ns) 826 return nil 827 case "Timestamp": 828 unq, err := unquote(string(inputValue)) 829 if err != nil { 830 return err 831 } 832 833 t, err := time.Parse(time.RFC3339Nano, unq) 834 if err != nil { 835 return fmt.Errorf("bad Timestamp: %v", err) 836 } 837 838 target.Field(0).SetInt(t.Unix()) 839 target.Field(1).SetInt(int64(t.Nanosecond())) 840 return nil 841 case "Struct": 842 var m map[string]json.RawMessage 843 if err := json.Unmarshal(inputValue, &m); err != nil { 844 return fmt.Errorf("bad StructValue: %v", err) 845 } 846 847 target.Field(0).Set(reflect.ValueOf(map[string]*stpb.Value{})) 848 for k, jv := range m { 849 pv := &stpb.Value{} 850 if err := u.unmarshalValue(reflect.ValueOf(pv).Elem(), jv, prop); err != nil { 851 return fmt.Errorf("bad value in StructValue for key %q: %v", k, err) 852 } 853 target.Field(0).SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(pv)) 854 } 855 return nil 856 case "ListValue": 857 var s []json.RawMessage 858 if err := json.Unmarshal(inputValue, &s); err != nil { 859 return fmt.Errorf("bad ListValue: %v", err) 860 } 861 862 target.Field(0).Set(reflect.ValueOf(make([]*stpb.Value, len(s)))) 863 for i, sv := range s { 864 if err := u.unmarshalValue(target.Field(0).Index(i), sv, prop); err != nil { 865 return err 866 } 867 } 868 return nil 869 case "Value": 870 ivStr := string(inputValue) 871 if ivStr == "null" { 872 target.Field(0).Set(reflect.ValueOf(&stpb.Value_NullValue{})) 873 } else if v, err := strconv.ParseFloat(ivStr, 0); err == nil { 874 target.Field(0).Set(reflect.ValueOf(&stpb.Value_NumberValue{v})) 875 } else if v, err := unquote(ivStr); err == nil { 876 target.Field(0).Set(reflect.ValueOf(&stpb.Value_StringValue{v})) 877 } else if v, err := strconv.ParseBool(ivStr); err == nil { 878 target.Field(0).Set(reflect.ValueOf(&stpb.Value_BoolValue{v})) 879 } else if err := json.Unmarshal(inputValue, &[]json.RawMessage{}); err == nil { 880 lv := &stpb.ListValue{} 881 target.Field(0).Set(reflect.ValueOf(&stpb.Value_ListValue{lv})) 882 return u.unmarshalValue(reflect.ValueOf(lv).Elem(), inputValue, prop) 883 } else if err := json.Unmarshal(inputValue, &map[string]json.RawMessage{}); err == nil { 884 sv := &stpb.Struct{} 885 target.Field(0).Set(reflect.ValueOf(&stpb.Value_StructValue{sv})) 886 return u.unmarshalValue(reflect.ValueOf(sv).Elem(), inputValue, prop) 887 } else { 888 return fmt.Errorf("unrecognized type for Value %q", ivStr) 889 } 890 return nil 891 } 892 } 893 894 // Handle enums, which have an underlying type of int32, 895 // and may appear as strings. 896 // The case of an enum appearing as a number is handled 897 // at the bottom of this function. 898 if inputValue[0] == '"' && prop != nil && prop.Enum != "" { 899 vmap := proto.EnumValueMap(prop.Enum) 900 // Don't need to do unquoting; valid enum names 901 // are from a limited character set. 902 s := inputValue[1 : len(inputValue)-1] 903 n, ok := vmap[string(s)] 904 if !ok { 905 return fmt.Errorf("unknown value %q for enum %s", s, prop.Enum) 906 } 907 if target.Kind() == reflect.Ptr { // proto2 908 target.Set(reflect.New(targetType.Elem())) 909 target = target.Elem() 910 } 911 if targetType.Kind() != reflect.Int32 { 912 return fmt.Errorf("invalid target %q for enum %s", targetType.Kind(), prop.Enum) 913 } 914 target.SetInt(int64(n)) 915 return nil 916 } 917 918 // Handle nested messages. 919 if targetType.Kind() == reflect.Struct { 920 var jsonFields map[string]json.RawMessage 921 if err := json.Unmarshal(inputValue, &jsonFields); err != nil { 922 return err 923 } 924 925 consumeField := func(prop *proto.Properties) (json.RawMessage, bool) { 926 // Be liberal in what names we accept; both orig_name and camelName are okay. 927 fieldNames := acceptedJSONFieldNames(prop) 928 929 vOrig, okOrig := jsonFields[fieldNames.orig] 930 vCamel, okCamel := jsonFields[fieldNames.camel] 931 if !okOrig && !okCamel { 932 return nil, false 933 } 934 // If, for some reason, both are present in the data, favour the camelName. 935 var raw json.RawMessage 936 if okOrig { 937 raw = vOrig 938 delete(jsonFields, fieldNames.orig) 939 } 940 if okCamel { 941 raw = vCamel 942 delete(jsonFields, fieldNames.camel) 943 } 944 return raw, true 945 } 946 947 sprops := proto.GetProperties(targetType) 948 for i := 0; i < target.NumField(); i++ { 949 ft := target.Type().Field(i) 950 if strings.HasPrefix(ft.Name, "XXX_") { 951 continue 952 } 953 954 valueForField, ok := consumeField(sprops.Prop[i]) 955 if !ok { 956 continue 957 } 958 959 if err := u.unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil { 960 return err 961 } 962 } 963 // Check for any oneof fields. 964 if len(jsonFields) > 0 { 965 for _, oop := range sprops.OneofTypes { 966 raw, ok := consumeField(oop.Prop) 967 if !ok { 968 continue 969 } 970 nv := reflect.New(oop.Type.Elem()) 971 target.Field(oop.Field).Set(nv) 972 if err := u.unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil { 973 return err 974 } 975 } 976 } 977 // Handle proto2 extensions. 978 if len(jsonFields) > 0 { 979 if ep, ok := target.Addr().Interface().(proto.Message); ok { 980 for _, ext := range proto.RegisteredExtensions(ep) { 981 name := fmt.Sprintf("[%s]", ext.Name) 982 raw, ok := jsonFields[name] 983 if !ok { 984 continue 985 } 986 delete(jsonFields, name) 987 nv := reflect.New(reflect.TypeOf(ext.ExtensionType).Elem()) 988 if err := u.unmarshalValue(nv.Elem(), raw, nil); err != nil { 989 return err 990 } 991 if err := proto.SetExtension(ep, ext, nv.Interface()); err != nil { 992 return err 993 } 994 } 995 } 996 } 997 if !u.AllowUnknownFields && len(jsonFields) > 0 { 998 // Pick any field to be the scapegoat. 999 var f string 1000 for fname := range jsonFields { 1001 f = fname 1002 break 1003 } 1004 return fmt.Errorf("unknown field %q in %v", f, targetType) 1005 } 1006 return nil 1007 } 1008 1009 // Handle arrays (which aren't encoded bytes) 1010 if targetType.Kind() == reflect.Slice && targetType.Elem().Kind() != reflect.Uint8 { 1011 var slc []json.RawMessage 1012 if err := json.Unmarshal(inputValue, &slc); err != nil { 1013 return err 1014 } 1015 if slc != nil { 1016 l := len(slc) 1017 target.Set(reflect.MakeSlice(targetType, l, l)) 1018 for i := 0; i < l; i++ { 1019 if err := u.unmarshalValue(target.Index(i), slc[i], prop); err != nil { 1020 return err 1021 } 1022 } 1023 } 1024 return nil 1025 } 1026 1027 // Handle maps (whose keys are always strings) 1028 if targetType.Kind() == reflect.Map { 1029 var mp map[string]json.RawMessage 1030 if err := json.Unmarshal(inputValue, &mp); err != nil { 1031 return err 1032 } 1033 if mp != nil { 1034 target.Set(reflect.MakeMap(targetType)) 1035 for ks, raw := range mp { 1036 // Unmarshal map key. The core json library already decoded the key into a 1037 // string, so we handle that specially. Other types were quoted post-serialization. 1038 var k reflect.Value 1039 if targetType.Key().Kind() == reflect.String { 1040 k = reflect.ValueOf(ks) 1041 } else { 1042 k = reflect.New(targetType.Key()).Elem() 1043 var kprop *proto.Properties 1044 if prop != nil && prop.MapKeyProp != nil { 1045 kprop = prop.MapKeyProp 1046 } 1047 if err := u.unmarshalValue(k, json.RawMessage(ks), kprop); err != nil { 1048 return err 1049 } 1050 } 1051 1052 // Unmarshal map value. 1053 v := reflect.New(targetType.Elem()).Elem() 1054 var vprop *proto.Properties 1055 if prop != nil && prop.MapValProp != nil { 1056 vprop = prop.MapValProp 1057 } 1058 if err := u.unmarshalValue(v, raw, vprop); err != nil { 1059 return err 1060 } 1061 target.SetMapIndex(k, v) 1062 } 1063 } 1064 return nil 1065 } 1066 1067 // Non-finite numbers can be encoded as strings. 1068 isFloat := targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64 1069 if isFloat { 1070 if num, ok := nonFinite[string(inputValue)]; ok { 1071 target.SetFloat(num) 1072 return nil 1073 } 1074 } 1075 1076 // integers & floats can be encoded as strings. In this case we drop 1077 // the quotes and proceed as normal. 1078 isNum := targetType.Kind() == reflect.Int64 || targetType.Kind() == reflect.Uint64 || 1079 targetType.Kind() == reflect.Int32 || targetType.Kind() == reflect.Uint32 || 1080 targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64 1081 if isNum && strings.HasPrefix(string(inputValue), `"`) { 1082 inputValue = inputValue[1 : len(inputValue)-1] 1083 } 1084 1085 // Use the encoding/json for parsing other value types. 1086 return json.Unmarshal(inputValue, target.Addr().Interface()) 1087} 1088 1089func unquote(s string) (string, error) { 1090 var ret string 1091 err := json.Unmarshal([]byte(s), &ret) 1092 return ret, err 1093} 1094 1095// jsonProperties returns parsed proto.Properties for the field and corrects JSONName attribute. 1096func jsonProperties(f reflect.StructField, origName bool) *proto.Properties { 1097 var prop proto.Properties 1098 prop.Init(f.Type, f.Name, f.Tag.Get("protobuf"), &f) 1099 if origName || prop.JSONName == "" { 1100 prop.JSONName = prop.OrigName 1101 } 1102 return &prop 1103} 1104 1105type fieldNames struct { 1106 orig, camel string 1107} 1108 1109func acceptedJSONFieldNames(prop *proto.Properties) fieldNames { 1110 opts := fieldNames{orig: prop.OrigName, camel: prop.OrigName} 1111 if prop.JSONName != "" { 1112 opts.camel = prop.JSONName 1113 } 1114 return opts 1115} 1116 1117// Writer wrapper inspired by https://blog.golang.org/errors-are-values 1118type errWriter struct { 1119 writer io.Writer 1120 err error 1121} 1122 1123func (w *errWriter) write(str string) { 1124 if w.err != nil { 1125 return 1126 } 1127 _, w.err = w.writer.Write([]byte(str)) 1128} 1129 1130// Map fields may have key types of non-float scalars, strings and enums. 1131// The easiest way to sort them in some deterministic order is to use fmt. 1132// If this turns out to be inefficient we can always consider other options, 1133// such as doing a Schwartzian transform. 1134// 1135// Numeric keys are sorted in numeric order per 1136// https://developers.google.com/protocol-buffers/docs/proto#maps. 1137type mapKeys []reflect.Value 1138 1139func (s mapKeys) Len() int { return len(s) } 1140func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 1141func (s mapKeys) Less(i, j int) bool { 1142 if k := s[i].Kind(); k == s[j].Kind() { 1143 switch k { 1144 case reflect.String: 1145 return s[i].String() < s[j].String() 1146 case reflect.Int32, reflect.Int64: 1147 return s[i].Int() < s[j].Int() 1148 case reflect.Uint32, reflect.Uint64: 1149 return s[i].Uint() < s[j].Uint() 1150 } 1151 } 1152 return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface()) 1153} 1154 1155// checkRequiredFields returns an error if any required field in the given proto message is not set. 1156// This function is used by both Marshal and Unmarshal. While required fields only exist in a 1157// proto2 message, a proto3 message can contain proto2 message(s). 1158func checkRequiredFields(pb proto.Message) error { 1159 // Most well-known type messages do not contain required fields. The "Any" type may contain 1160 // a message that has required fields. 1161 // 1162 // When an Any message is being marshaled, the code will invoked proto.Unmarshal on Any.Value 1163 // field in order to transform that into JSON, and that should have returned an error if a 1164 // required field is not set in the embedded message. 1165 // 1166 // When an Any message is being unmarshaled, the code will have invoked proto.Marshal on the 1167 // embedded message to store the serialized message in Any.Value field, and that should have 1168 // returned an error if a required field is not set. 1169 if _, ok := pb.(wkt); ok { 1170 return nil 1171 } 1172 1173 v := reflect.ValueOf(pb) 1174 // Skip message if it is not a struct pointer. 1175 if v.Kind() != reflect.Ptr { 1176 return nil 1177 } 1178 v = v.Elem() 1179 if v.Kind() != reflect.Struct { 1180 return nil 1181 } 1182 1183 for i := 0; i < v.NumField(); i++ { 1184 field := v.Field(i) 1185 sfield := v.Type().Field(i) 1186 1187 if sfield.PkgPath != "" { 1188 // blank PkgPath means the field is exported; skip if not exported 1189 continue 1190 } 1191 1192 if strings.HasPrefix(sfield.Name, "XXX_") { 1193 continue 1194 } 1195 1196 // Oneof field is an interface implemented by wrapper structs containing the actual oneof 1197 // field, i.e. an interface containing &T{real_value}. 1198 if sfield.Tag.Get("protobuf_oneof") != "" { 1199 if field.Kind() != reflect.Interface { 1200 continue 1201 } 1202 v := field.Elem() 1203 if v.Kind() != reflect.Ptr || v.IsNil() { 1204 continue 1205 } 1206 v = v.Elem() 1207 if v.Kind() != reflect.Struct || v.NumField() < 1 { 1208 continue 1209 } 1210 field = v.Field(0) 1211 sfield = v.Type().Field(0) 1212 } 1213 1214 protoTag := sfield.Tag.Get("protobuf") 1215 if protoTag == "" { 1216 continue 1217 } 1218 var prop proto.Properties 1219 prop.Init(sfield.Type, sfield.Name, protoTag, &sfield) 1220 1221 switch field.Kind() { 1222 case reflect.Map: 1223 if field.IsNil() { 1224 continue 1225 } 1226 // Check each map value. 1227 keys := field.MapKeys() 1228 for _, k := range keys { 1229 v := field.MapIndex(k) 1230 if err := checkRequiredFieldsInValue(v); err != nil { 1231 return err 1232 } 1233 } 1234 case reflect.Slice: 1235 // Handle non-repeated type, e.g. bytes. 1236 if !prop.Repeated { 1237 if prop.Required && field.IsNil() { 1238 return fmt.Errorf("required field %q is not set", prop.Name) 1239 } 1240 continue 1241 } 1242 1243 // Handle repeated type. 1244 if field.IsNil() { 1245 continue 1246 } 1247 // Check each slice item. 1248 for i := 0; i < field.Len(); i++ { 1249 v := field.Index(i) 1250 if err := checkRequiredFieldsInValue(v); err != nil { 1251 return err 1252 } 1253 } 1254 case reflect.Ptr: 1255 if field.IsNil() { 1256 if prop.Required { 1257 return fmt.Errorf("required field %q is not set", prop.Name) 1258 } 1259 continue 1260 } 1261 if err := checkRequiredFieldsInValue(field); err != nil { 1262 return err 1263 } 1264 } 1265 } 1266 1267 // Handle proto2 extensions. 1268 for _, ext := range proto.RegisteredExtensions(pb) { 1269 if !proto.HasExtension(pb, ext) { 1270 continue 1271 } 1272 ep, err := proto.GetExtension(pb, ext) 1273 if err != nil { 1274 return err 1275 } 1276 err = checkRequiredFieldsInValue(reflect.ValueOf(ep)) 1277 if err != nil { 1278 return err 1279 } 1280 } 1281 1282 return nil 1283} 1284 1285func checkRequiredFieldsInValue(v reflect.Value) error { 1286 if v.Type().Implements(messageType) { 1287 return checkRequiredFields(v.Interface().(proto.Message)) 1288 } 1289 return nil 1290} 1291