• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python3
2#from calendar import c
3import sys
4import os
5import copy
6import argparse
7import statistics
8import glob
9import subprocess
10import re
11import time
12
13from string import digits
14
15class LogLine:
16	remove_digits = str.maketrans('', '', digits)
17	def __init__(self):
18		self.lineNum = 0
19		self.timeStamp = 0
20		self.delta = 0
21		self.deltaDiff = 0
22		self.text = "none"
23		self.textKey = "none"
24
25	def parse(self, index, line, priorTimeStamp):
26		_line = line.strip()
27		words = _line.split("]", 1)
28		timeString = words[0].strip(" [")
29		self.lineNum = index
30		self.timeStamp = float(timeString)
31		self.delta = self.timeStamp - priorTimeStamp
32		self.text = words[1][:150]
33		self.textKey = self.text.translate(self.remove_digits)
34		priorTimeStamp = self.timeStamp
35		return self
36
37	def getTextKey(self):
38		textKey = self.textKey
39		return textKey
40
41	def print(self):
42		print("I, {:5d}, T, {:8.4f}, D, {: .4f}, DD, ({: .4f}) {}".format(self.lineNum, self.timeStamp, self.delta, self.deltaDiff, self.text))
43
44	def toString(self):
45		return "I, {:5d}, T, {:8.4f}, D, {: .4f}, DD, ({: .4f}) {}".format(self.lineNum, self.timeStamp, self.delta, self.deltaDiff, self.text)
46
47def sortByDelta(item):
48	return item.delta
49
50def sortByTimeStamp(item):
51	return item.timeStamp
52
53class LogLineListStats:
54	def __init__(self):
55		self.numItems = 0
56		self.firstTimeStamp = 0
57		self.lastTimeStamp = 0
58		self.deltaSum = 0
59		self.deltaDiffSum = 0
60		self.status = "unknown"
61		self.name = "unknown"
62
63	def print(self):
64		print("Name {:25} NumItems {:4d} FirstTimeStamp {:.3f}, lastTimeStamp {:.3f}, deltaTime {:.3f} deltaSum {:.3f}, deltaDiffSum {:.3f} Status {}".format(self.name, self.numItems, self.firstTimeStamp, self.lastTimeStamp, (self.lastTimeStamp - self.firstTimeStamp), self.deltaSum, self.deltaDiffSum, self.status))
65
66	def add(self, _other):
67		if (_other.firstTimeStamp< self.firstTimeStamp):
68				self.firstTimeStamp = _other.firstTimeStamp
69
70		if (_other.lastTimeStamp > self.lastTimeStamp):
71			self.lastTimeStamp = _other.lastTimeStamp
72		self.deltaSum += _other.deltaSum
73
74
75# ------------------------------------------------------
76
77class LogLineList:
78
79	def __init__(self, _name= ""):
80		self.list = []
81		self.name = _name
82
83	def clear(self):
84		self.list.clear()
85
86	def append(self, item):
87		self.list.append(item)
88
89	def print(self, numItems=None):
90		printLineNum = 0
91		timeStart = 0
92		sumDelta = 0
93		sumDeltaDiff = 0
94		print("List: {}", self.name)
95		for item in self.list:
96			if (timeStart==0):
97				timeStart = item.timeStamp
98			timeOffset = item.timeStamp - timeStart
99			sumDelta += item.delta
100			sumDeltaDiff += item.deltaDiff
101			printLineNum += 1
102			printLine = "{:4d} {:.4f} {: .4f} ({: .4f}) | {} ".format(printLineNum, timeOffset, sumDelta, sumDeltaDiff, item.toString())
103			print(printLine)
104			if (numItems!=None):
105				numItems -= 1
106				if (numItems<=0):
107					break
108
109	def find(self, word):
110		itemList = []
111		for item in self.list:
112			if item.text.find(word) != -1:
113				itemList.append(item)
114		return itemList
115	def findFirst(self, word):
116		itemList = self.find(word)
117		if (itemList!=None):
118			if (len(itemList)>0):
119				return itemList[0]
120		return None
121
122	def findTextKey(self, textKey):
123		itemList = []
124		for item in self.list:
125			if item.getTextKey()==textKey:
126				itemList.append(item)
127		if (len(itemList)==0):
128			return None
129		return itemList[0]
130
131	def findItem(self, item):
132		textKey = item.getTextKey()
133		return self.findTextKey(textKey)
134
135	def findExactItem(self, item):
136		text = item.text
137		return self.find(text)
138
139	def filter(self, startKeyWord, endKeyWord, delta=0):
140		resultsList = LogLineList()
141		startTime = self.findFirst(startKeyWord).timeStamp
142		endTime = self.findFirst(endKeyWord).timeStamp
143		for item in self.list:
144			if ((item.timeStamp >= startTime) and (item.timeStamp<=endTime)):
145				if (item.timeStamp == startTime):
146					item.delta = 0
147				if ((item.delta > delta) or ((item.timeStamp == startTime))):
148					resultsList.append(item)
149		resultsList.name = self.name
150		return resultsList
151
152
153	def findCommon(self, otherList):
154		commonList = LogLineList()
155		commonList.name = self.name + "_common"
156		notCommonList = LogLineList()
157		notCommonList.name = self.name + "_notCommon"
158		numFoundItems = 0
159		numNotFoundItems = 0
160		for item in self.list:
161			dm1 = otherList.findExactItem(item)
162			_item = copy.deepcopy(item)
163			if dm1!=None:
164				commonList.append(_item)
165				numFoundItems += 1
166			else:
167				notCommonList.append(_item)
168				numNotFoundItems += 1
169		print("FindCommon {} {} {} {}".format(len(self.list), len(otherList.list), numFoundItems, numNotFoundItems  ))
170		return commonList, notCommonList
171
172	def difference(self, otherList):
173		diffList = LogLineList()
174		diffList.name = otherList.name + "Diff"
175		for item in self.list:
176			thisItem = copy.deepcopy(item)
177			otherItem = otherList.findItem(item)
178			if (item.text.find("EXT4-fs (sda11): recovery complete")!=-1):
179				print("here")
180			if otherItem==None:
181				print("LogLineList::difference() !Error, other does not have {}".format(item.text))
182			else:
183				thisItem.deltaDiff = otherItem.delta - item.delta
184
185			diffList.append(thisItem)
186		return diffList
187
188	def analyze(self, checkPeriod = True, includeFirst = True):
189		numItems = 0
190		firstTimeStamp = 0
191		firstDelta = 0
192		lastTimeStamp = 0
193		deltaSum = 0
194		deltaDiffSum = 0
195		for item in self.list:
196			numItems += 1
197			deltaSum += item.delta
198			deltaDiffSum += item.deltaDiff
199			if firstTimeStamp==0:
200				firstTimeStamp = item.timeStamp
201				firstDelta = item.delta
202				deltaSum = 0
203				deltaDiffSum = 0
204			if (item.timeStamp<firstTimeStamp):
205				firstTimeStamp = item.timeStamp
206				firstDelta = item.delta
207
208			if (item.timeStamp > lastTimeStamp):
209				lastTimeStamp = item.timeStamp
210		timePeriod = lastTimeStamp - firstTimeStamp
211		status = "pass"
212		if (checkPeriod):
213			diff = timePeriod - deltaSum
214			if (abs(diff)>0.0001):
215				print("LogLineList::Analyze() {} ERROR! TimePeriod:{}, CumulativeDelta: {} ".format(self.name, timePeriod, deltaSum))
216				status = "ERROR"
217		logLineListStats = LogLineListStats()
218		logLineListStats.numItems = numItems
219		logLineListStats.firstTimeStamp = firstTimeStamp
220		logLineListStats.lastTimeStamp = lastTimeStamp
221		logLineListStats.deltaSum = deltaSum
222		logLineListStats.deltaDiffSum = deltaDiffSum
223		logLineListStats.status = status
224		logLineListStats.name = self.name
225		return logLineListStats
226
227	def addList(self, otherList):
228		self.list.extend(otherList.list)
229		self.list = sorted(self.list, key=sortByTimeStamp)
230
231
232class LogFile:
233	priorTimeStamp = 0.0
234	def __init__(self, _fileName = ""):
235		self.logLineList = LogLineList()
236		if (_fileName!=""):
237			self.load(_fileName)
238
239	def loadLines(self, lines):
240		logLineList = LogLineList()
241		for index, line in enumerate(lines):
242			logLine = LogLine().parse(index, line, self.priorTimeStamp)
243			self.priorTimeStamp = logLine.timeStamp
244			logLineList.append(logLine)
245		return logLineList
246
247	def load(self, _fileName):
248		self.name = _fileName
249		try:
250			file = open(_fileName, 'r')
251			lines = file.readlines()
252			self.logLineList = self.loadLines(lines)
253			file.close()
254		except:
255			print("Error, file '{}' does not exist".format(self.name))
256
257	def print(self, numItems=None):
258		self.logLineList.print(numItems)
259
260# -----------------------------------------------------
261
262class MetricSet:
263	def __init__(self, _names):
264		self.columnNames = _names
265		self.rowColArray = []
266		self.rowSum = []
267		self.rowMax = []
268		self.rowMin = []
269		self.rowStd = []
270		self.rowMedian = []
271		for col in self.columnNames:
272			self.rowSum.append(0)
273			self.rowMax.append(0)
274			self.rowMin.append(sys.maxsize)
275			self.rowStd.append(0)
276			self.rowMedian.append(0)
277
278	def appendSet(self, values):
279		self.rowColArray.append(values)
280
281	def print(self):
282		print("{}".format("  Line#"), end='')
283		for words in self.columnNames:
284			print(", '{}'".format(words), end='')
285		print("")
286
287		for row, values in enumerate(self.rowColArray):
288			print("{:6d}".format(row), end='')
289			for col, value in enumerate(values):
290				print(", {:.3f}".format(value), end='')
291			print("")
292
293		print("{}".format("   MAX"), end='')
294		for value in self.rowMax:
295			print(", {:.3f}".format(value), end='')
296		print("")
297
298
299		print("{}".format("   AVE"), end='')
300		for value in self.rowSum:
301			print(", {:.3f}".format(value), end='')
302		print("")
303
304		print("{}".format("   MIN"), end='')
305		for value in self.rowMin:
306			print(", {:2.3f}".format(value), end='')
307		print("")
308
309		print("{}".format("   STD"), end='')
310		for value in self.rowStd:
311			print(", {:2.3f}".format(value), end='')
312		print("")
313
314		print("{}".format("MEDIAN"), end='')
315		for value in self.rowMedian:
316			print(", {:2.3f}".format(value), end='')
317		print("")
318
319	def analyze(self):
320		stdCols = []
321		numCols = len(self.columnNames)
322		numRows = len(self.rowColArray)
323		for col in range(numCols):
324			stdCols.append([])
325
326		# compute sum
327		for row, values in enumerate(self.rowColArray):
328			for col, value in enumerate(values):
329				self.rowSum[col] += value
330				if value > self.rowMax[col]:
331					self.rowMax[col] = value
332				if value < self.rowMin[col]:
333					self.rowMin[col] = value
334
335		# compute std
336		for col in range(numCols):
337			for row in range(numRows):
338				try:
339					val = self.rowColArray[row][col]
340					stdCols[col].append(val)
341				except IndexError:
342					i = 3
343		for col, colList in enumerate(stdCols):
344			stdValue = 0
345			if (len(colList)>0):
346				stdValue = statistics.pstdev(colList)
347				stdMedian = statistics.median(colList)
348			self.rowStd[col] = stdValue
349			self.rowMedian[col] = stdMedian
350
351		#compute average
352		for col, value in enumerate(self.rowSum):
353			if numRows > 0:
354				self.rowSum[col] = self.rowSum[col] / numRows
355			else:
356				self.rowSum[col] = 0
357
358class AnalyzeFile:
359	initFirstTime = 0
360	initSecondTime = 0
361
362	def __init__(self, _fileName, _keyWords = ["init first", "init second", "boot_completed"]):
363		self.fileName = _fileName
364		self.logFile = LogFile(_fileName)
365		self.workingList = []
366		self.keyWords = _keyWords
367
368	def report(self):
369		print("-----------------------")
370		print("Reporting on '{}'".format(self.fileName))
371		for word in self.keyWords:
372			item = self.logFile.logLineList.findFirst(word)
373			item.print()
374		print("-----------------------")
375
376	def getMetrics(self, metricsSet):
377		values = []
378		for word in self.keyWords:
379			item = self.logFile.logLineList.findFirst(word)
380			if item is not None:
381				values.append(item.timeStamp)
382			else:
383				print("Did not find {} ".format(word))
384		metricsSet.appendSet(values)
385
386	def keyWordReport(self, keyword):
387		numItems = 0
388		cumd = 0
389		items = self.logFile.logLineList.find(keyword)
390		for item in items:
391			item.print()
392			numItems += 1
393			cumd += item.delta
394		print("Num {} found = {}, Sum delay = {:.2f} ".format(keyword, numItems, cumd))
395
396		for item in items:
397			lineKeywords = item.text.split(" ")
398			if (len(lineKeywords)>2):
399				if lineKeywords[2] == "Service":
400					tookIndex = item.text.find("took")
401					if (tookIndex!=None):
402						tookTime = item.text[tookIndex:tookIndex+10]
403						print("{} took {}".format(lineKeywords[3], tookTime))
404
405
406class Analyzer:
407	def __init__(self):
408		self.fileName = []
409
410	def rebootAndRunCmdToFile(self, fileNamePrefix, msgPrefix, Cmd, numTimes, startIndex):
411		captured = False
412		error = False
413		filenameNum = ""
414		for i in range(numTimes):
415			postfix = str(i+startIndex)
416			filenameNum = fileNamePrefix + "-" + postfix + ".txt"
417			print(msgPrefix + " to {}".format(filenameNum))
418			# try 5 times to capure status 'boot_completed'
419			for i in range(5):
420				captured = False
421				rebootCmd = "adb shell su root reboot"
422				fullCmd = Cmd + " > {}".format(filenameNum)
423				x = os.system(rebootCmd)
424				if (x!=0):
425					print("Error")
426					error = True
427					break
428				time.sleep(45)
429				x = os.system(fullCmd)
430				if (x!=0):
431					print("Error")
432					error = True
433					break
434				# check for boot complete
435				try:
436					checkBootComplete = "grep boot_complete {}".format(filenameNum)
437					output = subprocess.check_output(checkBootComplete, shell=True)
438					captured = True
439					break
440				except:
441					captured = False
442					print("trying again for {}".format(filenameNum))
443			if not captured:
444				print("ERROR - failed to capture {}".format(filenameNum))
445		if error:
446			os.system("rm {}".format(filenameNum))
447		return captured
448
449	def getBuildID(self):
450		buildIDCmd = "adb shell su root getprop ro.build.version.incremental"
451		buildString = subprocess.check_output(buildIDCmd, shell = True)
452		numberList = re.findall(r'\d+', buildString.decode('ISO-8859-1') )
453		if (numberList==None): return 0
454		if (len(numberList)==0): return 0
455		buildID = numberList[0]
456		return buildID
457
458	def pullDmesgLogs(self, BuildID, numTimes, startIndex):
459		fileNamePrefix = BuildID
460		msgPrefix = "Pulling Kernel dmesg logs"
461		cmd = "adb shell su root dmesg"
462		return self.rebootAndRunCmdToFile(fileNamePrefix, msgPrefix, cmd, numTimes, startIndex)
463
464	def pullLogcatLogs(self, BuildID, numTimes, startIndex):
465		fileNamePrefix = "LC-"+BuildID
466		msgPrefix = "Pulling Kernel Logcat"
467		cmd = "adb logcat -b all -d"
468		return self.rebootAndRunCmdToFile(fileNamePrefix, msgPrefix, cmd, numTimes, startIndex)
469
470	def runBootAnalyze(self, filename, numTimes, startIndex):
471		ABT = os.environ["ANDROID_BUILD_TOP"]
472		if (len(ABT)<=0):
473			print("ERROR - ANDROID_BUILD_TOP not set")
474		BAFILE = "BA-" + filename + "-" + str(numTimes + startIndex) + ".txt"
475		BACmd = ABT + "/system/extras/boottime_tools/bootanalyze/bootanalyze.py -c " + ABT + "/system/extras/boottime_tools/bootanalyze/config.yaml -n 20 -r -t > " + BAFILE
476		print(BACmd)
477		x = os.system(BACmd)
478		if (x!=0):
479			print("ERROR running bootanalze")
480			return False
481		return True
482
483	def pullAll(self):
484		BuildID = self.getBuildID()
485		Cmd = "adb bugreport bugreport-{}".format(BuildID)
486		print(Cmd)
487		x = os.system(Cmd)
488		if (x!=0):
489			print("ERROR Pulling all data")
490			return False
491		self.pullDmesgLogs(BuildID, 20, 0)
492		self.pullLogcatLogs(BuildID, 2, 0)
493		self.runBootAnalyze(BuildID, 20, 0)
494		self.summaryReportOnDmesgLogFiles(BuildID, 20)
495
496	def summaryReportOnDmesgLogFiles(self, BuildID, numFiles):
497		metricKeyWords = ["init first", "init second", "boot_completed"]
498		metricSet = MetricSet(metricKeyWords)
499		print("Summary report on log files with build ID {}".format(BuildID))
500		dirList = glob.glob("{}*.txt".format(BuildID))
501		numFilesAnalyzed = 0
502		for index, file in enumerate(dirList):
503			analyzeFile = AnalyzeFile(file, metricKeyWords)
504			#check it's a kernel log file
505			item = analyzeFile.logFile.logLineList.findFirst("build.fingerprint")
506			if (item!=None):
507				#check if it has the correct build ID
508				if (item.text.find(BuildID)==-1):
509					continue
510			else:
511				print("BuildID {} not found in file {} fingerprint {}".format(BuildID, file, item))
512				continue
513			analyzeFile.getMetrics(metricSet)
514			numFilesAnalyzed += 1
515			if ((index+1)>=numFiles):
516				break
517		if (numFilesAnalyzed>0):
518			metricSet.analyze()
519			metricSet.print()
520		else:
521			print("No files criteria {}* and build.fingerprint with {}".format(BuildID, BuildID))
522
523	def rename(self, BuildID1, BuildID2, fileType):
524		print("Summary report on log files with build ID {}".format(BuildID1))
525		dirList = glob.glob("*{}*".format(BuildID1))
526		for index, file in enumerate(dirList):
527			findRes = file.find(BuildID1)
528			if (findRes!=-1):
529				newFile = file.replace(BuildID1, BuildID2, 1)
530				newFile += fileType
531				os.system("mv {} {}".format(file, newFile))
532
533
534parser = argparse.ArgumentParser(description='pull all data files from seahawk and run dmesg summary report. The data files will be prefixed with the build ID')
535
536parser.add_argument("-plc", nargs=3, metavar=('<BuildID>', '<numTimes>', '<startIndex>'),  help="pull logcat numTimes from seahawk")
537parser.add_argument("-pdm", nargs=3, metavar=('<BuildID>', '<numTimes>', '<startIndex>'),  help="pull dmesg logs numTimes from seahawk")
538parser.add_argument("-pba", nargs=2, metavar=('<BuildID>', '<numTimes>'),  help="pull bootanalyze numTimes from seahawk")
539parser.add_argument("-rd", nargs=2, metavar=('<BuildID>', '<numFiles>'),  help="summary report on <numFiles> dmesg log files named <BuildID>-*.txt in current directory")
540parser.add_argument("-pA", action='store_true', help="pull all data from seahawk a default number of times")
541parser.add_argument("-t", nargs="*", help="test - do not use")
542args = parser.parse_args()
543
544
545if args.pdm!=None:
546	Analyzer().pullDmesgLogs(args.pdm[0], int(args.pdm[1]), int(args.pdm[2]))
547
548if args.plc!=None:
549	Analyzer().pullLogcatLogs(args.plc[0], int(args.plc[1]), int(args.plc[2]))
550
551if args.pba!=None:
552	Analyzer().runBootAnalyze(args.pba[0], int(args.pba[1]), 0)
553
554if args.pA!=None:
555	Analyzer().pullAll()
556
557if args.rd!=None:
558	Analyzer().summaryReportOnDmesgLogFiles(args.rd[0], int(args.rd[1]))
559
560if args.t!=None:
561	Analyzer().getBuildID()
562
563