1// Copyright 2011 Google Inc. All rights reserved. 2// Use of this source code is governed by the Apache 2.0 3// license that can be found in the LICENSE file. 4 5package datastore 6 7import ( 8 "fmt" 9 "reflect" 10 "strings" 11 "time" 12 13 "github.com/golang/protobuf/proto" 14 "google.golang.org/appengine" 15 pb "google.golang.org/appengine/internal/datastore" 16) 17 18var ( 19 typeOfBlobKey = reflect.TypeOf(appengine.BlobKey("")) 20 typeOfByteSlice = reflect.TypeOf([]byte(nil)) 21 typeOfByteString = reflect.TypeOf(ByteString(nil)) 22 typeOfGeoPoint = reflect.TypeOf(appengine.GeoPoint{}) 23 typeOfTime = reflect.TypeOf(time.Time{}) 24 typeOfKeyPtr = reflect.TypeOf(&Key{}) 25 typeOfEntityPtr = reflect.TypeOf(&Entity{}) 26) 27 28// typeMismatchReason returns a string explaining why the property p could not 29// be stored in an entity field of type v.Type(). 30func typeMismatchReason(pValue interface{}, v reflect.Value) string { 31 entityType := "empty" 32 switch pValue.(type) { 33 case int64: 34 entityType = "int" 35 case bool: 36 entityType = "bool" 37 case string: 38 entityType = "string" 39 case float64: 40 entityType = "float" 41 case *Key: 42 entityType = "*datastore.Key" 43 case time.Time: 44 entityType = "time.Time" 45 case appengine.BlobKey: 46 entityType = "appengine.BlobKey" 47 case appengine.GeoPoint: 48 entityType = "appengine.GeoPoint" 49 case ByteString: 50 entityType = "datastore.ByteString" 51 case []byte: 52 entityType = "[]byte" 53 } 54 return fmt.Sprintf("type mismatch: %s versus %v", entityType, v.Type()) 55} 56 57type propertyLoader struct { 58 // m holds the number of times a substruct field like "Foo.Bar.Baz" has 59 // been seen so far. The map is constructed lazily. 60 m map[string]int 61} 62 63func (l *propertyLoader) load(codec *structCodec, structValue reflect.Value, p Property, requireSlice bool) string { 64 var v reflect.Value 65 var sliceIndex int 66 67 name := p.Name 68 69 // If name ends with a '.', the last field is anonymous. 70 // In this case, strings.Split will give us "" as the 71 // last element of our fields slice, which will match the "" 72 // field name in the substruct codec. 73 fields := strings.Split(name, ".") 74 75 for len(fields) > 0 { 76 var decoder fieldCodec 77 var ok bool 78 79 // Cut off the last field (delimited by ".") and find its parent 80 // in the codec. 81 // eg. for name "A.B.C.D", split off "A.B.C" and try to 82 // find a field in the codec with this name. 83 // Loop again with "A.B", etc. 84 for i := len(fields); i > 0; i-- { 85 parent := strings.Join(fields[:i], ".") 86 decoder, ok = codec.fields[parent] 87 if ok { 88 fields = fields[i:] 89 break 90 } 91 } 92 93 // If we never found a matching field in the codec, return 94 // error message. 95 if !ok { 96 return "no such struct field" 97 } 98 99 v = initField(structValue, decoder.path) 100 if !v.IsValid() { 101 return "no such struct field" 102 } 103 if !v.CanSet() { 104 return "cannot set struct field" 105 } 106 107 if decoder.structCodec != nil { 108 codec = decoder.structCodec 109 structValue = v 110 } 111 112 if v.Kind() == reflect.Slice && v.Type() != typeOfByteSlice { 113 if l.m == nil { 114 l.m = make(map[string]int) 115 } 116 sliceIndex = l.m[p.Name] 117 l.m[p.Name] = sliceIndex + 1 118 for v.Len() <= sliceIndex { 119 v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem())) 120 } 121 structValue = v.Index(sliceIndex) 122 requireSlice = false 123 } 124 } 125 126 var slice reflect.Value 127 if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 { 128 slice = v 129 v = reflect.New(v.Type().Elem()).Elem() 130 } else if requireSlice { 131 return "multiple-valued property requires a slice field type" 132 } 133 134 // Convert indexValues to a Go value with a meaning derived from the 135 // destination type. 136 pValue := p.Value 137 if iv, ok := pValue.(indexValue); ok { 138 meaning := pb.Property_NO_MEANING 139 switch v.Type() { 140 case typeOfBlobKey: 141 meaning = pb.Property_BLOBKEY 142 case typeOfByteSlice: 143 meaning = pb.Property_BLOB 144 case typeOfByteString: 145 meaning = pb.Property_BYTESTRING 146 case typeOfGeoPoint: 147 meaning = pb.Property_GEORSS_POINT 148 case typeOfTime: 149 meaning = pb.Property_GD_WHEN 150 case typeOfEntityPtr: 151 meaning = pb.Property_ENTITY_PROTO 152 } 153 var err error 154 pValue, err = propValue(iv.value, meaning) 155 if err != nil { 156 return err.Error() 157 } 158 } 159 160 if errReason := setVal(v, pValue); errReason != "" { 161 // Set the slice back to its zero value. 162 if slice.IsValid() { 163 slice.Set(reflect.Zero(slice.Type())) 164 } 165 return errReason 166 } 167 168 if slice.IsValid() { 169 slice.Index(sliceIndex).Set(v) 170 } 171 172 return "" 173} 174 175// setVal sets v to the value pValue. 176func setVal(v reflect.Value, pValue interface{}) string { 177 switch v.Kind() { 178 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 179 x, ok := pValue.(int64) 180 if !ok && pValue != nil { 181 return typeMismatchReason(pValue, v) 182 } 183 if v.OverflowInt(x) { 184 return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type()) 185 } 186 v.SetInt(x) 187 case reflect.Bool: 188 x, ok := pValue.(bool) 189 if !ok && pValue != nil { 190 return typeMismatchReason(pValue, v) 191 } 192 v.SetBool(x) 193 case reflect.String: 194 switch x := pValue.(type) { 195 case appengine.BlobKey: 196 v.SetString(string(x)) 197 case ByteString: 198 v.SetString(string(x)) 199 case string: 200 v.SetString(x) 201 default: 202 if pValue != nil { 203 return typeMismatchReason(pValue, v) 204 } 205 } 206 case reflect.Float32, reflect.Float64: 207 x, ok := pValue.(float64) 208 if !ok && pValue != nil { 209 return typeMismatchReason(pValue, v) 210 } 211 if v.OverflowFloat(x) { 212 return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type()) 213 } 214 v.SetFloat(x) 215 case reflect.Ptr: 216 x, ok := pValue.(*Key) 217 if !ok && pValue != nil { 218 return typeMismatchReason(pValue, v) 219 } 220 if _, ok := v.Interface().(*Key); !ok { 221 return typeMismatchReason(pValue, v) 222 } 223 v.Set(reflect.ValueOf(x)) 224 case reflect.Struct: 225 switch v.Type() { 226 case typeOfTime: 227 x, ok := pValue.(time.Time) 228 if !ok && pValue != nil { 229 return typeMismatchReason(pValue, v) 230 } 231 v.Set(reflect.ValueOf(x)) 232 case typeOfGeoPoint: 233 x, ok := pValue.(appengine.GeoPoint) 234 if !ok && pValue != nil { 235 return typeMismatchReason(pValue, v) 236 } 237 v.Set(reflect.ValueOf(x)) 238 default: 239 ent, ok := pValue.(*Entity) 240 if !ok { 241 return typeMismatchReason(pValue, v) 242 } 243 244 // Recursively load nested struct 245 pls, err := newStructPLS(v.Addr().Interface()) 246 if err != nil { 247 return err.Error() 248 } 249 250 // if ent has a Key value and our struct has a Key field, 251 // load the Entity's Key value into the Key field on the struct. 252 if ent.Key != nil && pls.codec.keyField != -1 { 253 254 pls.v.Field(pls.codec.keyField).Set(reflect.ValueOf(ent.Key)) 255 } 256 257 err = pls.Load(ent.Properties) 258 if err != nil { 259 return err.Error() 260 } 261 } 262 case reflect.Slice: 263 x, ok := pValue.([]byte) 264 if !ok { 265 if y, yok := pValue.(ByteString); yok { 266 x, ok = []byte(y), true 267 } 268 } 269 if !ok && pValue != nil { 270 return typeMismatchReason(pValue, v) 271 } 272 if v.Type().Elem().Kind() != reflect.Uint8 { 273 return typeMismatchReason(pValue, v) 274 } 275 v.SetBytes(x) 276 default: 277 return typeMismatchReason(pValue, v) 278 } 279 return "" 280} 281 282// initField is similar to reflect's Value.FieldByIndex, in that it 283// returns the nested struct field corresponding to index, but it 284// initialises any nil pointers encountered when traversing the structure. 285func initField(val reflect.Value, index []int) reflect.Value { 286 for _, i := range index[:len(index)-1] { 287 val = val.Field(i) 288 if val.Kind() == reflect.Ptr { 289 if val.IsNil() { 290 val.Set(reflect.New(val.Type().Elem())) 291 } 292 val = val.Elem() 293 } 294 } 295 return val.Field(index[len(index)-1]) 296} 297 298// loadEntity loads an EntityProto into PropertyLoadSaver or struct pointer. 299func loadEntity(dst interface{}, src *pb.EntityProto) (err error) { 300 ent, err := protoToEntity(src) 301 if err != nil { 302 return err 303 } 304 if e, ok := dst.(PropertyLoadSaver); ok { 305 return e.Load(ent.Properties) 306 } 307 return LoadStruct(dst, ent.Properties) 308} 309 310func (s structPLS) Load(props []Property) error { 311 var fieldName, reason string 312 var l propertyLoader 313 for _, p := range props { 314 if errStr := l.load(s.codec, s.v, p, p.Multiple); errStr != "" { 315 // We don't return early, as we try to load as many properties as possible. 316 // It is valid to load an entity into a struct that cannot fully represent it. 317 // That case returns an error, but the caller is free to ignore it. 318 fieldName, reason = p.Name, errStr 319 } 320 } 321 if reason != "" { 322 return &ErrFieldMismatch{ 323 StructType: s.v.Type(), 324 FieldName: fieldName, 325 Reason: reason, 326 } 327 } 328 return nil 329} 330 331func protoToEntity(src *pb.EntityProto) (*Entity, error) { 332 props, rawProps := src.Property, src.RawProperty 333 outProps := make([]Property, 0, len(props)+len(rawProps)) 334 for { 335 var ( 336 x *pb.Property 337 noIndex bool 338 ) 339 if len(props) > 0 { 340 x, props = props[0], props[1:] 341 } else if len(rawProps) > 0 { 342 x, rawProps = rawProps[0], rawProps[1:] 343 noIndex = true 344 } else { 345 break 346 } 347 348 var value interface{} 349 if x.Meaning != nil && *x.Meaning == pb.Property_INDEX_VALUE { 350 value = indexValue{x.Value} 351 } else { 352 var err error 353 value, err = propValue(x.Value, x.GetMeaning()) 354 if err != nil { 355 return nil, err 356 } 357 } 358 outProps = append(outProps, Property{ 359 Name: x.GetName(), 360 Value: value, 361 NoIndex: noIndex, 362 Multiple: x.GetMultiple(), 363 }) 364 } 365 366 var key *Key 367 if src.Key != nil { 368 // Ignore any error, since nested entity values 369 // are allowed to have an invalid key. 370 key, _ = protoToKey(src.Key) 371 } 372 return &Entity{key, outProps}, nil 373} 374 375// propValue returns a Go value that combines the raw PropertyValue with a 376// meaning. For example, an Int64Value with GD_WHEN becomes a time.Time. 377func propValue(v *pb.PropertyValue, m pb.Property_Meaning) (interface{}, error) { 378 switch { 379 case v.Int64Value != nil: 380 if m == pb.Property_GD_WHEN { 381 return fromUnixMicro(*v.Int64Value), nil 382 } else { 383 return *v.Int64Value, nil 384 } 385 case v.BooleanValue != nil: 386 return *v.BooleanValue, nil 387 case v.StringValue != nil: 388 if m == pb.Property_BLOB { 389 return []byte(*v.StringValue), nil 390 } else if m == pb.Property_BLOBKEY { 391 return appengine.BlobKey(*v.StringValue), nil 392 } else if m == pb.Property_BYTESTRING { 393 return ByteString(*v.StringValue), nil 394 } else if m == pb.Property_ENTITY_PROTO { 395 var ent pb.EntityProto 396 err := proto.Unmarshal([]byte(*v.StringValue), &ent) 397 if err != nil { 398 return nil, err 399 } 400 return protoToEntity(&ent) 401 } else { 402 return *v.StringValue, nil 403 } 404 case v.DoubleValue != nil: 405 return *v.DoubleValue, nil 406 case v.Referencevalue != nil: 407 key, err := referenceValueToKey(v.Referencevalue) 408 if err != nil { 409 return nil, err 410 } 411 return key, nil 412 case v.Pointvalue != nil: 413 // NOTE: Strangely, latitude maps to X, longitude to Y. 414 return appengine.GeoPoint{Lat: v.Pointvalue.GetX(), Lng: v.Pointvalue.GetY()}, nil 415 } 416 return nil, nil 417} 418 419// indexValue is a Property value that is created when entities are loaded from 420// an index, such as from a projection query. 421// 422// Such Property values do not contain all of the metadata required to be 423// faithfully represented as a Go value, and are instead represented as an 424// opaque indexValue. Load the properties into a concrete struct type (e.g. by 425// passing a struct pointer to Iterator.Next) to reconstruct actual Go values 426// of type int, string, time.Time, etc. 427type indexValue struct { 428 value *pb.PropertyValue 429} 430