Lines Matching full:token
397 def push(self, token): argument
398 self.tokens.insert(0, token);
401 print("Last token: ", self.last)
402 print("Token queue: ", self.tokens)
405 def token(self): member in CLexer
626 def error(self, msg, token=-1): argument
631 if token != -1:
632 print("Got token ", token)
636 def debug(self, msg, token=-1): argument
638 if token != -1:
639 print("Got token ", token)
670 def parseComment(self, token): argument
672 self.top_comment = token[1]
673 if self.comment == None or token[1][0] == '*':
674 self.comment = token[1];
676 self.comment = self.comment + token[1]
677 token = self.lexer.token()
685 return token
916 def parsePreproc(self, token): argument
918 print("=> preproc ", token, self.lexer.tokens)
919 name = token[1]
921 token = self.lexer.token()
922 if token == None:
924 if token[0] == 'preproc':
925 self.index_add(token[1], self.filename, not self.is_header,
927 return self.lexer.token()
928 return token
930 token = self.lexer.token()
931 if token == None:
933 if token[0] == 'preproc':
935 name = token[1]
937 token = self.lexer.token()
938 while token != None and token[0] == 'preproc' and \
939 token[1][0] != '#':
940 lst.append(token[1])
941 token = self.lexer.token()
949 return token
1005 token = self.lexer.token()
1006 while token != None and token[0] == 'preproc' and \
1007 token[1][0] != '#':
1008 token = self.lexer.token()
1009 return token
1012 # token acquisition on top of the lexer, it handle internally
1016 def token(self): member in CParser
1019 token = self.lexer.token()
1020 while token != None:
1021 if token[0] == 'comment':
1022 token = self.parseComment(token)
1024 elif token[0] == 'preproc':
1025 token = self.parsePreproc(token)
1027 elif token[0] == "name" and token[1] == "__const":
1028 token = ("name", "const")
1029 return token
1030 elif token[0] == "name" and token[1] == "__attribute":
1031 token = self.lexer.token()
1032 while token != None and token[1] != ";":
1033 token = self.lexer.token()
1034 return token
1035 elif token[0] == "name" and token[1] in ignored_words:
1036 (n, info) = ignored_words[token[1]]
1039 token = self.lexer.token()
1041 token = self.lexer.token()
1045 print("=> ", token)
1046 return token
1052 def parseTypedef(self, token): argument
1053 if token == None:
1055 token = self.parseType(token)
1056 if token == None:
1061 #self.debug("end typedef type", token)
1062 while token != None:
1063 if token[0] == "name":
1064 name = token[1]
1082 token = self.token()
1085 return token
1086 #self.debug("end typedef", token)
1087 if token != None and token[0] == 'sep' and token[1] == ',':
1089 token = self.token()
1090 while token != None and token[0] == "op":
1091 type = type + token[1]
1092 token = self.token()
1093 elif token != None and token[0] == 'sep' and token[1] == ';':
1095 elif token != None and token[0] == 'name':
1099 self.error("parsing typedef: expecting ';'", token)
1100 return token
1101 token = self.token()
1102 return token
1108 def parseBlock(self, token): argument
1109 while token != None:
1110 if token[0] == "sep" and token[1] == "{":
1111 token = self.token()
1112 token = self.parseBlock(token)
1113 elif token[0] == "sep" and token[1] == "}":
1114 token = self.token()
1115 return token
1118 oldtok = token
1119 token = self.token()
1121 if token[0] == "sep" and token[1] == "(":
1124 token = self.token()
1125 elif token[0] == "name":
1126 token = self.token()
1127 if token[0] == "sep" and (token[1] == ";" or
1128 token[1] == "," or token[1] == "="):
1139 token = self.token()
1140 return token
1145 def parseStruct(self, token): argument
1147 #self.debug("start parseStruct", token)
1148 while token != None:
1149 if token[0] == "sep" and token[1] == "{":
1150 token = self.token()
1151 token = self.parseTypeBlock(token)
1152 elif token[0] == "sep" and token[1] == "}":
1154 #self.debug("end parseStruct", token)
1156 token = self.token()
1157 return token
1160 #self.debug("before parseType", token)
1161 token = self.parseType(token)
1162 #self.debug("after parseType", token)
1163 if token != None and token[0] == "name":
1164 fname = token[1]
1165 token = self.token()
1166 if token[0] == "sep" and token[1] == ";":
1167 token = self.token()
1170 self.error("parseStruct: expecting ;", token)
1171 elif token != None and token[0] == "sep" and token[1] == "{":
1172 token = self.token()
1173 token = self.parseTypeBlock(token)
1174 if token != None and token[0] == "name":
1175 token = self.token()
1176 if token != None and token[0] == "sep" and token[1] == ";":
1177 token = self.token()
1179 self.error("parseStruct: expecting ;", token)
1181 self.error("parseStruct: name", token)
1182 token = self.token()
1185 #self.debug("end parseStruct", token)
1187 return token
1192 def parseEnumBlock(self, token): argument
1198 while token != None:
1199 if token[0] == "sep" and token[1] == "{":
1200 token = self.token()
1201 token = self.parseTypeBlock(token)
1202 elif token[0] == "sep" and token[1] == "}":
1208 token = self.token()
1209 return token
1210 elif token[0] == "name":
1216 name = token[1]
1218 token = self.token()
1219 if token[0] == "op" and token[1][0] == "=":
1221 if len(token[1]) > 1:
1222 value = token[1][1:]
1223 token = self.token()
1224 while token[0] != "sep" or (token[1] != ',' and
1225 token[1] != '}'):
1226 value = value + token[1]
1227 token = self.token()
1234 if token[0] == "sep" and token[1] == ",":
1235 token = self.token()
1237 token = self.token()
1238 return token
1244 def parseTypeBlock(self, token): argument
1245 while token != None:
1246 if token[0] == "sep" and token[1] == "{":
1247 token = self.token()
1248 token = self.parseTypeBlock(token)
1249 elif token[0] == "sep" and token[1] == "}":
1250 token = self.token()
1251 return token
1253 token = self.token()
1254 return token
1259 # if inside, the name token is pushed back before returning
1261 def parseType(self, token): argument
1265 if token == None:
1266 return token
1271 while token[0] == "name" and (
1272 token[1] == "const" or \
1273 token[1] == "unsigned" or \
1274 token[1] == "signed"):
1275 if token[1] == "unsigned" or token[1] == "signed":
1278 self.type = token[1]
1280 self.type = self.type + " " + token[1]
1281 token = self.token()
1283 if token[0] == "name" and token[1] in ("char", "short", "int", "long"):
1285 self.type = token[1]
1287 self.type = self.type + " " + token[1]
1292 elif token[0] == "name" and token[1] == "struct":
1294 self.type = token[1]
1296 self.type = self.type + " " + token[1]
1297 token = self.token()
1299 if token[0] == "name":
1300 nametok = token
1301 token = self.token()
1302 if token != None and token[0] == "sep" and token[1] == "{":
1303 token = self.token()
1304 token = self.parseStruct(token)
1305 elif token != None and token[0] == "op" and token[1] == "*":
1307 token = self.token()
1308 while token != None and token[0] == "op" and token[1] == "*":
1310 token = self.token()
1311 if token[0] == "name":
1312 nametok = token
1313 token = self.token()
1315 self.error("struct : expecting name", token)
1316 return token
1317 elif token != None and token[0] == "name" and nametok != None:
1319 return token
1322 self.lexer.push(token)
1323 token = nametok
1324 return token
1326 elif token[0] == "name" and token[1] == "enum":
1328 self.type = token[1]
1330 self.type = self.type + " " + token[1]
1332 token = self.token()
1333 if token != None and token[0] == "sep" and token[1] == "{":
1334 token = self.token()
1335 token = self.parseEnumBlock(token)
1337 self.error("parsing enum: expecting '{'", token)
1339 if token != None and token[0] != "name":
1340 self.lexer.push(token)
1341 token = ("name", "enum")
1343 enum_type = token[1]
1348 return token
1350 elif token[0] == "name":
1352 self.type = token[1]
1354 self.type = self.type + " " + token[1]
1357 token)
1358 return token
1360 token = self.token()
1361 while token != None and (token[0] == "op" or
1362 token[0] == "name" and token[1] == "const"):
1363 self.type = self.type + " " + token[1]
1364 token = self.token()
1369 if token != None and token[0] == "sep" and token[1] == '(':
1370 self.type = self.type + token[1]
1371 token = self.token()
1372 while token != None and token[0] == "op" and token[1] == '*':
1373 self.type = self.type + token[1]
1374 token = self.token()
1375 if token == None or token[0] != "name" :
1376 self.error("parsing function type, name expected", token);
1377 return token
1378 self.type = self.type + token[1]
1379 nametok = token
1380 token = self.token()
1381 if token != None and token[0] == "sep" and token[1] == ')':
1382 self.type = self.type + token[1]
1383 token = self.token()
1384 if token != None and token[0] == "sep" and token[1] == '(':
1385 token = self.token()
1387 token = self.parseSignature(token);
1390 self.error("parsing function type, '(' expected", token);
1391 return token
1393 self.error("parsing function type, ')' expected", token);
1394 return token
1395 self.lexer.push(token)
1396 token = nametok
1397 return token
1402 if token != None and token[0] == "name":
1403 nametok = token
1404 token = self.token()
1405 if token != None and token[0] == "sep" and token[1] == '[':
1407 while token != None and token[0] == "sep" and token[1] == '[':
1408 self.type = self.type + token[1]
1409 token = self.token()
1410 while token != None and token[0] != 'sep' and \
1411 token[1] != ']' and token[1] != ';':
1412 self.type = self.type + token[1]
1413 token = self.token()
1414 if token != None and token[0] == 'sep' and token[1] == ']':
1415 self.type = self.type + token[1]
1416 token = self.token()
1418 self.error("parsing array type, ']' expected", token);
1419 return token
1420 elif token != None and token[0] == "sep" and token[1] == ':':
1422 token = self.token()
1423 token = self.token()
1424 self.lexer.push(token)
1425 token = nametok
1427 return token
1432 def parseSignature(self, token): argument
1434 if token != None and token[0] == "sep" and token[1] == ')':
1436 token = self.token()
1437 return token
1438 while token != None:
1439 token = self.parseType(token)
1440 if token != None and token[0] == "name":
1441 signature.append((self.type, token[1], None))
1442 token = self.token()
1443 elif token != None and token[0] == "sep" and token[1] == ',':
1444 token = self.token()
1446 elif token != None and token[0] == "sep" and token[1] == ')':
1452 if token != None and token[0] == "sep":
1453 if token[1] == ',':
1454 token = self.token()
1456 elif token[1] == ')':
1457 token = self.token()
1460 return token
1466 def parseGlobal(self, token): argument
1468 if token[1] == 'extern':
1469 token = self.token()
1470 if token == None:
1471 return token
1472 if token[0] == 'string':
1473 if token[1] == 'C':
1474 token = self.token()
1475 if token == None:
1476 return token
1477 if token[0] == 'sep' and token[1] == "{":
1478 token = self.token()
1480 while token != None and (token[0] != 'sep' or
1481 token[1] != "}"):
1482 if token[0] == 'name':
1483 token = self.parseGlobal(token)
1486 "token %s %s unexpected at the top level" % (
1487 token[0], token[1]))
1488 token = self.parseGlobal(token)
1490 token = self.token()
1491 return token
1493 return token
1494 elif token[1] == 'static':
1496 token = self.token()
1497 if token == None or token[0] != 'name':
1498 return token
1500 if token[1] == 'typedef':
1501 token = self.token()
1502 return self.parseTypedef(token)
1504 token = self.parseType(token)
1506 if token == None or token[0] != "name":
1507 return token
1509 self.name = token[1]
1510 token = self.token()
1511 while token != None and (token[0] == "sep" or token[0] == "op"):
1512 if token[0] == "sep":
1513 if token[1] == "[":
1514 type = type + token[1]
1515 token = self.token()
1516 while token != None and (token[0] != "sep" or \
1517 token[1] != ";"):
1518 type = type + token[1]
1519 token = self.token()
1521 if token != None and token[0] == "op" and token[1] == "=":
1525 token = self.token()
1526 if token[0] == 'sep' and token[1] == '{':
1527 token = self.token()
1528 token = self.parseBlock(token)
1530 while token != None and (token[0] != "sep" or \
1531 (token[1] != ';' and token[1] != ',')):
1532 token = self.token()
1533 if token == None or token[0] != "sep" or (token[1] != ';' and
1534 token[1] != ','):
1537 if token != None and token[0] == "sep":
1538 if token[1] == ";":
1547 token = self.token()
1549 elif token[1] == "(":
1550 token = self.token()
1551 token = self.parseSignature(token)
1552 if token == None:
1554 if token[0] == "sep" and token[1] == ";":
1560 token = self.token()
1561 elif token[0] == "sep" and token[1] == "{":
1567 token = self.token()
1568 token = self.parseBlock(token);
1569 elif token[1] == ',':
1574 token = self.token()
1575 while token != None and token[0] == "sep":
1576 type = type + token[1]
1577 token = self.token()
1578 if token != None and token[0] == "name":
1579 self.name = token[1]
1580 token = self.token()
1584 return token
1588 token = self.token()
1589 while token != None:
1590 if token[0] == 'name':
1591 token = self.parseGlobal(token)
1593 self.error("token %s %s unexpected at the top level" % (
1594 token[0], token[1]))
1595 token = self.parseGlobal(token)