Lines Matching full:token
394 def push(self, token): argument
395 self.tokens.insert(0, token);
398 print("Last token: ", self.last)
399 print("Token queue: ", self.tokens)
402 def token(self): member in CLexer
623 def error(self, msg, token=-1): argument
628 if token != -1:
629 print("Got token ", token)
633 def debug(self, msg, token=-1): argument
635 if token != -1:
636 print("Got token ", token)
667 def parseComment(self, token): argument
669 self.top_comment = token[1]
670 if self.comment == None or token[1][0] == '*':
671 self.comment = token[1];
673 self.comment = self.comment + token[1]
674 token = self.lexer.token()
682 return token
913 def parsePreproc(self, token): argument
915 print("=> preproc ", token, self.lexer.tokens)
916 name = token[1]
918 token = self.lexer.token()
919 if token == None:
921 if token[0] == 'preproc':
922 self.index_add(token[1], self.filename, not self.is_header,
924 return self.lexer.token()
925 return token
927 token = self.lexer.token()
928 if token == None:
930 if token[0] == 'preproc':
932 name = token[1]
934 token = self.lexer.token()
935 while token != None and token[0] == 'preproc' and \
936 token[1][0] != '#':
937 lst.append(token[1])
938 token = self.lexer.token()
946 return token
1002 token = self.lexer.token()
1003 while token != None and token[0] == 'preproc' and \
1004 token[1][0] != '#':
1005 token = self.lexer.token()
1006 return token
1009 # token acquisition on top of the lexer, it handle internally
1013 def token(self): member in CParser
1016 token = self.lexer.token()
1017 while token != None:
1018 if token[0] == 'comment':
1019 token = self.parseComment(token)
1021 elif token[0] == 'preproc':
1022 token = self.parsePreproc(token)
1024 elif token[0] == "name" and token[1] == "__const":
1025 token = ("name", "const")
1026 return token
1027 elif token[0] == "name" and token[1] == "__attribute":
1028 token = self.lexer.token()
1029 while token != None and token[1] != ";":
1030 token = self.lexer.token()
1031 return token
1032 elif token[0] == "name" and token[1] in ignored_words:
1033 (n, info) = ignored_words[token[1]]
1036 token = self.lexer.token()
1038 token = self.lexer.token()
1042 print("=> ", token)
1043 return token
1049 def parseTypedef(self, token): argument
1050 if token == None:
1052 token = self.parseType(token)
1053 if token == None:
1058 #self.debug("end typedef type", token)
1059 while token != None:
1060 if token[0] == "name":
1061 name = token[1]
1079 token = self.token()
1082 return token
1083 #self.debug("end typedef", token)
1084 if token != None and token[0] == 'sep' and token[1] == ',':
1086 token = self.token()
1087 while token != None and token[0] == "op":
1088 type = type + token[1]
1089 token = self.token()
1090 elif token != None and token[0] == 'sep' and token[1] == ';':
1092 elif token != None and token[0] == 'name':
1096 self.error("parsing typedef: expecting ';'", token)
1097 return token
1098 token = self.token()
1099 return token
1105 def parseBlock(self, token): argument
1106 while token != None:
1107 if token[0] == "sep" and token[1] == "{":
1108 token = self.token()
1109 token = self.parseBlock(token)
1110 elif token[0] == "sep" and token[1] == "}":
1111 token = self.token()
1112 return token
1115 oldtok = token
1116 token = self.token()
1118 if token[0] == "sep" and token[1] == "(":
1121 token = self.token()
1122 elif token[0] == "name":
1123 token = self.token()
1124 if token[0] == "sep" and (token[1] == ";" or
1125 token[1] == "," or token[1] == "="):
1136 token = self.token()
1137 return token
1142 def parseStruct(self, token): argument
1144 #self.debug("start parseStruct", token)
1145 while token != None:
1146 if token[0] == "sep" and token[1] == "{":
1147 token = self.token()
1148 token = self.parseTypeBlock(token)
1149 elif token[0] == "sep" and token[1] == "}":
1151 #self.debug("end parseStruct", token)
1153 token = self.token()
1154 return token
1157 #self.debug("before parseType", token)
1158 token = self.parseType(token)
1159 #self.debug("after parseType", token)
1160 if token != None and token[0] == "name":
1161 fname = token[1]
1162 token = self.token()
1163 if token[0] == "sep" and token[1] == ";":
1165 token = self.token()
1169 self.error("parseStruct: expecting ;", token)
1170 elif token != None and token[0] == "sep" and token[1] == "{":
1171 token = self.token()
1172 token = self.parseTypeBlock(token)
1173 if token != None and token[0] == "name":
1174 token = self.token()
1175 if token != None and token[0] == "sep" and token[1] == ";":
1176 token = self.token()
1178 self.error("parseStruct: expecting ;", token)
1180 self.error("parseStruct: name", token)
1181 token = self.token()
1184 #self.debug("end parseStruct", token)
1186 return token
1191 def parseEnumBlock(self, token): argument
1197 while token != None:
1198 if token[0] == "sep" and token[1] == "{":
1199 token = self.token()
1200 token = self.parseTypeBlock(token)
1201 elif token[0] == "sep" and token[1] == "}":
1207 token = self.token()
1208 return token
1209 elif token[0] == "name":
1215 name = token[1]
1217 token = self.token()
1218 if token[0] == "op" and token[1][0] == "=":
1220 if len(token[1]) > 1:
1221 value = token[1][1:]
1222 token = self.token()
1223 while token[0] != "sep" or (token[1] != ',' and
1224 token[1] != '}'):
1225 value = value + token[1]
1226 token = self.token()
1233 if token[0] == "sep" and token[1] == ",":
1234 token = self.token()
1236 token = self.token()
1237 return token
1243 def parseTypeBlock(self, token): argument
1244 while token != None:
1245 if token[0] == "sep" and token[1] == "{":
1246 token = self.token()
1247 token = self.parseTypeBlock(token)
1248 elif token[0] == "sep" and token[1] == "}":
1249 token = self.token()
1250 return token
1252 token = self.token()
1253 return token
1258 # if inside, the name token is pushed back before returning
1260 def parseType(self, token): argument
1264 if token == None:
1265 return token
1270 while token[0] == "name" and (
1271 token[1] == "const" or \
1272 token[1] == "unsigned" or \
1273 token[1] == "signed"):
1274 if token[1] == "unsigned" or token[1] == "signed":
1277 self.type = token[1]
1279 self.type = self.type + " " + token[1]
1280 token = self.token()
1282 if token[0] == "name" and token[1] in ("char", "short", "int", "long"):
1284 self.type = token[1]
1286 self.type = self.type + " " + token[1]
1291 elif token[0] == "name" and token[1] == "struct":
1293 self.type = token[1]
1295 self.type = self.type + " " + token[1]
1296 token = self.token()
1298 if token[0] == "name":
1299 nametok = token
1300 token = self.token()
1301 if token != None and token[0] == "sep" and token[1] == "{":
1302 token = self.token()
1303 token = self.parseStruct(token)
1304 elif token != None and token[0] == "op" and token[1] == "*":
1306 token = self.token()
1307 while token != None and token[0] == "op" and token[1] == "*":
1309 token = self.token()
1310 if token[0] == "name":
1311 nametok = token
1312 token = self.token()
1314 self.error("struct : expecting name", token)
1315 return token
1316 elif token != None and token[0] == "name" and nametok != None:
1318 return token
1321 self.lexer.push(token)
1322 token = nametok
1323 return token
1325 elif token[0] == "name" and token[1] == "enum":
1327 self.type = token[1]
1329 self.type = self.type + " " + token[1]
1331 token = self.token()
1332 if token != None and token[0] == "sep" and token[1] == "{":
1333 token = self.token()
1334 token = self.parseEnumBlock(token)
1336 self.error("parsing enum: expecting '{'", token)
1338 if token != None and token[0] != "name":
1339 self.lexer.push(token)
1340 token = ("name", "enum")
1342 enum_type = token[1]
1347 return token
1349 elif token[0] == "name":
1351 self.type = token[1]
1353 self.type = self.type + " " + token[1]
1356 token)
1357 return token
1359 token = self.token()
1360 while token != None and (token[0] == "op" or
1361 token[0] == "name" and token[1] == "const"):
1362 self.type = self.type + " " + token[1]
1363 token = self.token()
1368 if token != None and token[0] == "sep" and token[1] == '(':
1369 self.type = self.type + token[1]
1370 token = self.token()
1371 while token != None and token[0] == "op" and token[1] == '*':
1372 self.type = self.type + token[1]
1373 token = self.token()
1374 if token == None or token[0] != "name" :
1375 self.error("parsing function type, name expected", token);
1376 return token
1377 self.type = self.type + token[1]
1378 nametok = token
1379 token = self.token()
1380 if token != None and token[0] == "sep" and token[1] == ')':
1381 self.type = self.type + token[1]
1382 token = self.token()
1383 if token != None and token[0] == "sep" and token[1] == '(':
1384 token = self.token()
1386 token = self.parseSignature(token);
1389 self.error("parsing function type, '(' expected", token);
1390 return token
1392 self.error("parsing function type, ')' expected", token);
1393 return token
1394 self.lexer.push(token)
1395 token = nametok
1396 return token
1401 if token != None and token[0] == "name":
1402 nametok = token
1403 token = self.token()
1404 if token != None and token[0] == "sep" and token[1] == '[':
1406 while token != None and token[0] == "sep" and token[1] == '[':
1407 self.type = self.type + token[1]
1408 token = self.token()
1409 while token != None and token[0] != 'sep' and \
1410 token[1] != ']' and token[1] != ';':
1411 self.type = self.type + token[1]
1412 token = self.token()
1413 if token != None and token[0] == 'sep' and token[1] == ']':
1414 self.type = self.type + token[1]
1415 token = self.token()
1417 self.error("parsing array type, ']' expected", token);
1418 return token
1419 elif token != None and token[0] == "sep" and token[1] == ':':
1421 token = self.token()
1422 token = self.token()
1423 self.lexer.push(token)
1424 token = nametok
1426 return token
1431 def parseSignature(self, token): argument
1433 if token != None and token[0] == "sep" and token[1] == ')':
1435 token = self.token()
1436 return token
1437 while token != None:
1438 token = self.parseType(token)
1439 if token != None and token[0] == "name":
1440 signature.append((self.type, token[1], None))
1441 token = self.token()
1442 elif token != None and token[0] == "sep" and token[1] == ',':
1443 token = self.token()
1445 elif token != None and token[0] == "sep" and token[1] == ')':
1451 if token != None and token[0] == "sep":
1452 if token[1] == ',':
1453 token = self.token()
1455 elif token[1] == ')':
1456 token = self.token()
1459 return token
1465 def parseGlobal(self, token): argument
1467 if token[1] == 'extern':
1468 token = self.token()
1469 if token == None:
1470 return token
1471 if token[0] == 'string':
1472 if token[1] == 'C':
1473 token = self.token()
1474 if token == None:
1475 return token
1476 if token[0] == 'sep' and token[1] == "{":
1477 token = self.token()
1479 while token != None and (token[0] != 'sep' or
1480 token[1] != "}"):
1481 if token[0] == 'name':
1482 token = self.parseGlobal(token)
1485 "token %s %s unexpected at the top level" % (
1486 token[0], token[1]))
1487 token = self.parseGlobal(token)
1489 token = self.token()
1490 return token
1492 return token
1493 elif token[1] == 'static':
1495 token = self.token()
1496 if token == None or token[0] != 'name':
1497 return token
1499 if token[1] == 'typedef':
1500 token = self.token()
1501 return self.parseTypedef(token)
1503 token = self.parseType(token)
1505 if token == None or token[0] != "name":
1506 return token
1508 self.name = token[1]
1509 token = self.token()
1510 while token != None and (token[0] == "sep" or token[0] == "op"):
1511 if token[0] == "sep":
1512 if token[1] == "[":
1513 type = type + token[1]
1514 token = self.token()
1515 while token != None and (token[0] != "sep" or \
1516 token[1] != ";"):
1517 type = type + token[1]
1518 token = self.token()
1520 if token != None and token[0] == "op" and token[1] == "=":
1524 token = self.token()
1525 if token[0] == 'sep' and token[1] == '{':
1526 token = self.token()
1527 token = self.parseBlock(token)
1529 while token != None and (token[0] != "sep" or \
1530 (token[1] != ';' and token[1] != ',')):
1531 token = self.token()
1532 if token == None or token[0] != "sep" or (token[1] != ';' and
1533 token[1] != ','):
1536 if token != None and token[0] == "sep":
1537 if token[1] == ";":
1546 token = self.token()
1548 elif token[1] == "(":
1549 token = self.token()
1550 token = self.parseSignature(token)
1551 if token == None:
1553 if token[0] == "sep" and token[1] == ";":
1559 token = self.token()
1560 elif token[0] == "sep" and token[1] == "{":
1566 token = self.token()
1567 token = self.parseBlock(token);
1568 elif token[1] == ',':
1573 token = self.token()
1574 while token != None and token[0] == "sep":
1575 type = type + token[1]
1576 token = self.token()
1577 if token != None and token[0] == "name":
1578 self.name = token[1]
1579 token = self.token()
1583 return token
1587 token = self.token()
1588 while token != None:
1589 if token[0] == 'name':
1590 token = self.parseGlobal(token)
1592 self.error("token %s %s unexpected at the top level" % (
1593 token[0], token[1]))
1594 token = self.parseGlobal(token)