1// Copyright 2011 Google Inc. All rights reserved. 2// Use of this source code is governed by the Apache 2.0 3// license that can be found in the LICENSE file. 4 5/* 6Package log provides the means of writing and querying an application's logs 7from within an App Engine application. 8 9Example: 10 c := appengine.NewContext(r) 11 query := &log.Query{ 12 AppLogs: true, 13 Versions: []string{"1"}, 14 } 15 16 for results := query.Run(c); ; { 17 record, err := results.Next() 18 if err == log.Done { 19 log.Infof(c, "Done processing results") 20 break 21 } 22 if err != nil { 23 log.Errorf(c, "Failed to retrieve next log: %v", err) 24 break 25 } 26 log.Infof(c, "Saw record %v", record) 27 } 28*/ 29package log 30 31import ( 32 "errors" 33 "fmt" 34 "strings" 35 "time" 36 37 "github.com/golang/protobuf/proto" 38 "golang.org/x/net/context" 39 40 "google.golang.org/appengine" 41 "google.golang.org/appengine/internal" 42 pb "google.golang.org/appengine/internal/log" 43) 44 45// Query defines a logs query. 46type Query struct { 47 // Start time specifies the earliest log to return (inclusive). 48 StartTime time.Time 49 50 // End time specifies the latest log to return (exclusive). 51 EndTime time.Time 52 53 // Offset specifies a position within the log stream to resume reading from, 54 // and should come from a previously returned Record's field of the same name. 55 Offset []byte 56 57 // Incomplete controls whether active (incomplete) requests should be included. 58 Incomplete bool 59 60 // AppLogs indicates if application-level logs should be included. 61 AppLogs bool 62 63 // ApplyMinLevel indicates if MinLevel should be used to filter results. 64 ApplyMinLevel bool 65 66 // If ApplyMinLevel is true, only logs for requests with at least one 67 // application log of MinLevel or higher will be returned. 68 MinLevel int 69 70 // Versions is the major version IDs whose logs should be retrieved. 71 // Logs for specific modules can be retrieved by the specifying versions 72 // in the form "module:version"; the default module is used if no module 73 // is specified. 74 Versions []string 75 76 // A list of requests to search for instead of a time-based scan. Cannot be 77 // combined with filtering options such as StartTime, EndTime, Offset, 78 // Incomplete, ApplyMinLevel, or Versions. 79 RequestIDs []string 80} 81 82// AppLog represents a single application-level log. 83type AppLog struct { 84 Time time.Time 85 Level int 86 Message string 87} 88 89// Record contains all the information for a single web request. 90type Record struct { 91 AppID string 92 ModuleID string 93 VersionID string 94 RequestID []byte 95 IP string 96 Nickname string 97 AppEngineRelease string 98 99 // The time when this request started. 100 StartTime time.Time 101 102 // The time when this request finished. 103 EndTime time.Time 104 105 // Opaque cursor into the result stream. 106 Offset []byte 107 108 // The time required to process the request. 109 Latency time.Duration 110 MCycles int64 111 Method string 112 Resource string 113 HTTPVersion string 114 Status int32 115 116 // The size of the request sent back to the client, in bytes. 117 ResponseSize int64 118 Referrer string 119 UserAgent string 120 URLMapEntry string 121 Combined string 122 Host string 123 124 // The estimated cost of this request, in dollars. 125 Cost float64 126 TaskQueueName string 127 TaskName string 128 WasLoadingRequest bool 129 PendingTime time.Duration 130 Finished bool 131 AppLogs []AppLog 132 133 // Mostly-unique identifier for the instance that handled the request if available. 134 InstanceID string 135} 136 137// Result represents the result of a query. 138type Result struct { 139 logs []*Record 140 context context.Context 141 request *pb.LogReadRequest 142 resultsSeen bool 143 err error 144} 145 146// Next returns the next log record, 147func (qr *Result) Next() (*Record, error) { 148 if qr.err != nil { 149 return nil, qr.err 150 } 151 if len(qr.logs) > 0 { 152 lr := qr.logs[0] 153 qr.logs = qr.logs[1:] 154 return lr, nil 155 } 156 157 if qr.request.Offset == nil && qr.resultsSeen { 158 return nil, Done 159 } 160 161 if err := qr.run(); err != nil { 162 // Errors here may be retried, so don't store the error. 163 return nil, err 164 } 165 166 return qr.Next() 167} 168 169// Done is returned when a query iteration has completed. 170var Done = errors.New("log: query has no more results") 171 172// protoToAppLogs takes as input an array of pointers to LogLines, the internal 173// Protocol Buffer representation of a single application-level log, 174// and converts it to an array of AppLogs, the external representation 175// of an application-level log. 176func protoToAppLogs(logLines []*pb.LogLine) []AppLog { 177 appLogs := make([]AppLog, len(logLines)) 178 179 for i, line := range logLines { 180 appLogs[i] = AppLog{ 181 Time: time.Unix(0, *line.Time*1e3), 182 Level: int(*line.Level), 183 Message: *line.LogMessage, 184 } 185 } 186 187 return appLogs 188} 189 190// protoToRecord converts a RequestLog, the internal Protocol Buffer 191// representation of a single request-level log, to a Record, its 192// corresponding external representation. 193func protoToRecord(rl *pb.RequestLog) *Record { 194 offset, err := proto.Marshal(rl.Offset) 195 if err != nil { 196 offset = nil 197 } 198 return &Record{ 199 AppID: *rl.AppId, 200 ModuleID: rl.GetModuleId(), 201 VersionID: *rl.VersionId, 202 RequestID: rl.RequestId, 203 Offset: offset, 204 IP: *rl.Ip, 205 Nickname: rl.GetNickname(), 206 AppEngineRelease: string(rl.GetAppEngineRelease()), 207 StartTime: time.Unix(0, *rl.StartTime*1e3), 208 EndTime: time.Unix(0, *rl.EndTime*1e3), 209 Latency: time.Duration(*rl.Latency) * time.Microsecond, 210 MCycles: *rl.Mcycles, 211 Method: *rl.Method, 212 Resource: *rl.Resource, 213 HTTPVersion: *rl.HttpVersion, 214 Status: *rl.Status, 215 ResponseSize: *rl.ResponseSize, 216 Referrer: rl.GetReferrer(), 217 UserAgent: rl.GetUserAgent(), 218 URLMapEntry: *rl.UrlMapEntry, 219 Combined: *rl.Combined, 220 Host: rl.GetHost(), 221 Cost: rl.GetCost(), 222 TaskQueueName: rl.GetTaskQueueName(), 223 TaskName: rl.GetTaskName(), 224 WasLoadingRequest: rl.GetWasLoadingRequest(), 225 PendingTime: time.Duration(rl.GetPendingTime()) * time.Microsecond, 226 Finished: rl.GetFinished(), 227 AppLogs: protoToAppLogs(rl.Line), 228 InstanceID: string(rl.GetCloneKey()), 229 } 230} 231 232// Run starts a query for log records, which contain request and application 233// level log information. 234func (params *Query) Run(c context.Context) *Result { 235 req, err := makeRequest(params, internal.FullyQualifiedAppID(c), appengine.VersionID(c)) 236 return &Result{ 237 context: c, 238 request: req, 239 err: err, 240 } 241} 242 243func makeRequest(params *Query, appID, versionID string) (*pb.LogReadRequest, error) { 244 req := &pb.LogReadRequest{} 245 req.AppId = &appID 246 if !params.StartTime.IsZero() { 247 req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3) 248 } 249 if !params.EndTime.IsZero() { 250 req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3) 251 } 252 if len(params.Offset) > 0 { 253 var offset pb.LogOffset 254 if err := proto.Unmarshal(params.Offset, &offset); err != nil { 255 return nil, fmt.Errorf("bad Offset: %v", err) 256 } 257 req.Offset = &offset 258 } 259 if params.Incomplete { 260 req.IncludeIncomplete = ¶ms.Incomplete 261 } 262 if params.AppLogs { 263 req.IncludeAppLogs = ¶ms.AppLogs 264 } 265 if params.ApplyMinLevel { 266 req.MinimumLogLevel = proto.Int32(int32(params.MinLevel)) 267 } 268 if params.Versions == nil { 269 // If no versions were specified, default to the default module at 270 // the major version being used by this module. 271 if i := strings.Index(versionID, "."); i >= 0 { 272 versionID = versionID[:i] 273 } 274 req.VersionId = []string{versionID} 275 } else { 276 req.ModuleVersion = make([]*pb.LogModuleVersion, 0, len(params.Versions)) 277 for _, v := range params.Versions { 278 var m *string 279 if i := strings.Index(v, ":"); i >= 0 { 280 m, v = proto.String(v[:i]), v[i+1:] 281 } 282 req.ModuleVersion = append(req.ModuleVersion, &pb.LogModuleVersion{ 283 ModuleId: m, 284 VersionId: proto.String(v), 285 }) 286 } 287 } 288 if params.RequestIDs != nil { 289 ids := make([][]byte, len(params.RequestIDs)) 290 for i, v := range params.RequestIDs { 291 ids[i] = []byte(v) 292 } 293 req.RequestId = ids 294 } 295 296 return req, nil 297} 298 299// run takes the query Result produced by a call to Run and updates it with 300// more Records. The updated Result contains a new set of logs as well as an 301// offset to where more logs can be found. We also convert the items in the 302// response from their internal representations to external versions of the 303// same structs. 304func (r *Result) run() error { 305 res := &pb.LogReadResponse{} 306 if err := internal.Call(r.context, "logservice", "Read", r.request, res); err != nil { 307 return err 308 } 309 310 r.logs = make([]*Record, len(res.Log)) 311 r.request.Offset = res.Offset 312 r.resultsSeen = true 313 314 for i, log := range res.Log { 315 r.logs[i] = protoToRecord(log) 316 } 317 318 return nil 319} 320 321func init() { 322 internal.RegisterErrorCodeMap("logservice", pb.LogServiceError_ErrorCode_name) 323} 324