Lines Matching full:token
394 def push(self, token): argument
395 self.tokens.insert(0, token);
398 print("Last token: ", self.last)
399 print("Token queue: ", self.tokens)
402 def token(self): member in CLexer
623 def error(self, msg, token=-1): argument
628 if token != -1:
629 print("Got token ", token)
633 def debug(self, msg, token=-1): argument
635 if token != -1:
636 print("Got token ", token)
667 def parseComment(self, token): argument
669 self.top_comment = token[1]
670 if self.comment == None or token[1][0] == '*':
671 self.comment = token[1];
673 self.comment = self.comment + token[1]
674 token = self.lexer.token()
682 return token
913 def parsePreproc(self, token): argument
915 print("=> preproc ", token, self.lexer.tokens)
916 name = token[1]
918 token = self.lexer.token()
919 if token == None:
921 if token[0] == 'preproc':
922 self.index_add(token[1], self.filename, not self.is_header,
924 return self.lexer.token()
925 return token
927 token = self.lexer.token()
928 if token == None:
930 if token[0] == 'preproc':
932 name = token[1]
934 token = self.lexer.token()
935 while token != None and token[0] == 'preproc' and \
936 token[1][0] != '#':
937 lst.append(token[1])
938 token = self.lexer.token()
946 return token
1002 token = self.lexer.token()
1003 while token != None and token[0] == 'preproc' and \
1004 token[1][0] != '#':
1005 token = self.lexer.token()
1006 return token
1009 # token acquisition on top of the lexer, it handle internally
1013 def token(self): member in CParser
1016 token = self.lexer.token()
1017 while token != None:
1018 if token[0] == 'comment':
1019 token = self.parseComment(token)
1021 elif token[0] == 'preproc':
1022 token = self.parsePreproc(token)
1024 elif token[0] == "name" and token[1] == "__const":
1025 token = ("name", "const")
1026 return token
1027 elif token[0] == "name" and token[1] == "__attribute":
1028 token = self.lexer.token()
1029 while token != None and token[1] != ";":
1030 token = self.lexer.token()
1031 return token
1032 elif token[0] == "name" and token[1] in ignored_words:
1033 (n, info) = ignored_words[token[1]]
1036 token = self.lexer.token()
1038 token = self.lexer.token()
1042 print("=> ", token)
1043 return token
1049 def parseTypedef(self, token): argument
1050 if token == None:
1052 token = self.parseType(token)
1053 if token == None:
1058 #self.debug("end typedef type", token)
1059 while token != None:
1060 if token[0] == "name":
1061 name = token[1]
1079 token = self.token()
1082 return token
1083 #self.debug("end typedef", token)
1084 if token != None and token[0] == 'sep' and token[1] == ',':
1086 token = self.token()
1087 while token != None and token[0] == "op":
1088 type = type + token[1]
1089 token = self.token()
1090 elif token != None and token[0] == 'sep' and token[1] == ';':
1092 elif token != None and token[0] == 'name':
1096 self.error("parsing typedef: expecting ';'", token)
1097 return token
1098 token = self.token()
1099 return token
1105 def parseBlock(self, token): argument
1106 while token != None:
1107 if token[0] == "sep" and token[1] == "{":
1108 token = self.token()
1109 token = self.parseBlock(token)
1110 elif token[0] == "sep" and token[1] == "}":
1111 token = self.token()
1112 return token
1115 oldtok = token
1116 token = self.token()
1118 if token[0] == "sep" and token[1] == "(":
1121 token = self.token()
1122 elif token[0] == "name":
1123 token = self.token()
1124 if token[0] == "sep" and (token[1] == ";" or
1125 token[1] == "," or token[1] == "="):
1136 token = self.token()
1137 return token
1142 def parseStruct(self, token): argument
1144 #self.debug("start parseStruct", token)
1145 while token != None:
1146 if token[0] == "sep" and token[1] == "{":
1147 token = self.token()
1148 token = self.parseTypeBlock(token)
1149 elif token[0] == "sep" and token[1] == "}":
1151 #self.debug("end parseStruct", token)
1153 token = self.token()
1154 return token
1157 #self.debug("before parseType", token)
1158 token = self.parseType(token)
1159 #self.debug("after parseType", token)
1160 if token != None and token[0] == "name":
1161 fname = token[1]
1162 token = self.token()
1163 if token[0] == "sep" and token[1] == ";":
1165 token = self.token()
1169 self.error("parseStruct: expecting ;", token)
1170 elif token != None and token[0] == "sep" and token[1] == "{":
1171 token = self.token()
1172 token = self.parseTypeBlock(token)
1173 if token != None and token[0] == "name":
1174 token = self.token()
1175 if token != None and token[0] == "sep" and token[1] == ";":
1176 token = self.token()
1178 self.error("parseStruct: expecting ;", token)
1180 self.error("parseStruct: name", token)
1181 token = self.token()
1184 #self.debug("end parseStruct", token)
1186 return token
1191 def parseEnumBlock(self, token): argument
1197 while token != None:
1198 if token[0] == "sep" and token[1] == "{":
1199 token = self.token()
1200 token = self.parseTypeBlock(token)
1201 elif token[0] == "sep" and token[1] == "}":
1207 token = self.token()
1208 return token
1209 elif token[0] == "name":
1215 name = token[1]
1217 token = self.token()
1218 if token[0] == "op" and token[1][0] == "=":
1220 if len(token[1]) > 1:
1221 value = token[1][1:]
1222 token = self.token()
1223 while token[0] != "sep" or (token[1] != ',' and
1224 token[1] != '}'):
1225 value = value + token[1]
1226 token = self.token()
1233 if token[0] == "sep" and token[1] == ",":
1234 token = self.token()
1236 token = self.token()
1237 return token
1243 def parseTypeBlock(self, token): argument
1244 while token != None:
1245 if token[0] == "sep" and token[1] == "{":
1246 token = self.token()
1247 token = self.parseTypeBlock(token)
1248 elif token[0] == "sep" and token[1] == "}":
1249 token = self.token()
1250 return token
1252 token = self.token()
1253 return token
1258 # if inside, the name token is pushed back before returning
1260 def parseType(self, token): argument
1264 if token == None:
1265 return token
1267 while token[0] == "name" and (
1268 token[1] == "const" or \
1269 token[1] == "unsigned" or \
1270 token[1] == "signed"):
1272 self.type = token[1]
1274 self.type = self.type + " " + token[1]
1275 token = self.token()
1277 if token[0] == "name" and (token[1] == "long" or token[1] == "short"):
1279 self.type = token[1]
1281 self.type = self.type + " " + token[1]
1282 if token[0] == "name" and token[1] == "int":
1288 elif token[0] == "name" and token[1] == "struct":
1290 self.type = token[1]
1292 self.type = self.type + " " + token[1]
1293 token = self.token()
1295 if token[0] == "name":
1296 nametok = token
1297 token = self.token()
1298 if token != None and token[0] == "sep" and token[1] == "{":
1299 token = self.token()
1300 token = self.parseStruct(token)
1301 elif token != None and token[0] == "op" and token[1] == "*":
1303 token = self.token()
1304 while token != None and token[0] == "op" and token[1] == "*":
1306 token = self.token()
1307 if token[0] == "name":
1308 nametok = token
1309 token = self.token()
1311 self.error("struct : expecting name", token)
1312 return token
1313 elif token != None and token[0] == "name" and nametok != None:
1315 return token
1318 self.lexer.push(token)
1319 token = nametok
1320 return token
1322 elif token[0] == "name" and token[1] == "enum":
1324 self.type = token[1]
1326 self.type = self.type + " " + token[1]
1328 token = self.token()
1329 if token != None and token[0] == "sep" and token[1] == "{":
1330 token = self.token()
1331 token = self.parseEnumBlock(token)
1333 self.error("parsing enum: expecting '{'", token)
1335 if token != None and token[0] != "name":
1336 self.lexer.push(token)
1337 token = ("name", "enum")
1339 enum_type = token[1]
1344 return token
1346 elif token[0] == "name":
1348 self.type = token[1]
1350 self.type = self.type + " " + token[1]
1353 token)
1354 return token
1355 token = self.token()
1356 while token != None and (token[0] == "op" or
1357 token[0] == "name" and token[1] == "const"):
1358 self.type = self.type + " " + token[1]
1359 token = self.token()
1364 if token != None and token[0] == "sep" and token[1] == '(':
1365 self.type = self.type + token[1]
1366 token = self.token()
1367 while token != None and token[0] == "op" and token[1] == '*':
1368 self.type = self.type + token[1]
1369 token = self.token()
1370 if token == None or token[0] != "name" :
1371 self.error("parsing function type, name expected", token);
1372 return token
1373 self.type = self.type + token[1]
1374 nametok = token
1375 token = self.token()
1376 if token != None and token[0] == "sep" and token[1] == ')':
1377 self.type = self.type + token[1]
1378 token = self.token()
1379 if token != None and token[0] == "sep" and token[1] == '(':
1380 token = self.token()
1382 token = self.parseSignature(token);
1385 self.error("parsing function type, '(' expected", token);
1386 return token
1388 self.error("parsing function type, ')' expected", token);
1389 return token
1390 self.lexer.push(token)
1391 token = nametok
1392 return token
1397 if token != None and token[0] == "name":
1398 nametok = token
1399 token = self.token()
1400 if token != None and token[0] == "sep" and token[1] == '[':
1402 while token != None and token[0] == "sep" and token[1] == '[':
1403 self.type = self.type + token[1]
1404 token = self.token()
1405 while token != None and token[0] != 'sep' and \
1406 token[1] != ']' and token[1] != ';':
1407 self.type = self.type + token[1]
1408 token = self.token()
1409 if token != None and token[0] == 'sep' and token[1] == ']':
1410 self.type = self.type + token[1]
1411 token = self.token()
1413 self.error("parsing array type, ']' expected", token);
1414 return token
1415 elif token != None and token[0] == "sep" and token[1] == ':':
1417 token = self.token()
1418 token = self.token()
1419 self.lexer.push(token)
1420 token = nametok
1422 return token
1427 def parseSignature(self, token): argument
1429 if token != None and token[0] == "sep" and token[1] == ')':
1431 token = self.token()
1432 return token
1433 while token != None:
1434 token = self.parseType(token)
1435 if token != None and token[0] == "name":
1436 signature.append((self.type, token[1], None))
1437 token = self.token()
1438 elif token != None and token[0] == "sep" and token[1] == ',':
1439 token = self.token()
1441 elif token != None and token[0] == "sep" and token[1] == ')':
1447 if token != None and token[0] == "sep":
1448 if token[1] == ',':
1449 token = self.token()
1451 elif token[1] == ')':
1452 token = self.token()
1455 return token
1461 def parseGlobal(self, token): argument
1463 if token[1] == 'extern':
1464 token = self.token()
1465 if token == None:
1466 return token
1467 if token[0] == 'string':
1468 if token[1] == 'C':
1469 token = self.token()
1470 if token == None:
1471 return token
1472 if token[0] == 'sep' and token[1] == "{":
1473 token = self.token()
1475 while token != None and (token[0] != 'sep' or
1476 token[1] != "}"):
1477 if token[0] == 'name':
1478 token = self.parseGlobal(token)
1481 "token %s %s unexpected at the top level" % (
1482 token[0], token[1]))
1483 token = self.parseGlobal(token)
1485 token = self.token()
1486 return token
1488 return token
1489 elif token[1] == 'static':
1491 token = self.token()
1492 if token == None or token[0] != 'name':
1493 return token
1495 if token[1] == 'typedef':
1496 token = self.token()
1497 return self.parseTypedef(token)
1499 token = self.parseType(token)
1501 if token == None or token[0] != "name":
1502 return token
1504 self.name = token[1]
1505 token = self.token()
1506 while token != None and (token[0] == "sep" or token[0] == "op"):
1507 if token[0] == "sep":
1508 if token[1] == "[":
1509 type = type + token[1]
1510 token = self.token()
1511 while token != None and (token[0] != "sep" or \
1512 token[1] != ";"):
1513 type = type + token[1]
1514 token = self.token()
1516 if token != None and token[0] == "op" and token[1] == "=":
1520 token = self.token()
1521 if token[0] == 'sep' and token[1] == '{':
1522 token = self.token()
1523 token = self.parseBlock(token)
1525 while token != None and (token[0] != "sep" or \
1526 (token[1] != ';' and token[1] != ',')):
1527 token = self.token()
1528 if token == None or token[0] != "sep" or (token[1] != ';' and
1529 token[1] != ','):
1532 if token != None and token[0] == "sep":
1533 if token[1] == ";":
1542 token = self.token()
1544 elif token[1] == "(":
1545 token = self.token()
1546 token = self.parseSignature(token)
1547 if token == None:
1549 if token[0] == "sep" and token[1] == ";":
1555 token = self.token()
1556 elif token[0] == "sep" and token[1] == "{":
1562 token = self.token()
1563 token = self.parseBlock(token);
1564 elif token[1] == ',':
1569 token = self.token()
1570 while token != None and token[0] == "sep":
1571 type = type + token[1]
1572 token = self.token()
1573 if token != None and token[0] == "name":
1574 self.name = token[1]
1575 token = self.token()
1579 return token
1583 token = self.token()
1584 while token != None:
1585 if token[0] == 'name':
1586 token = self.parseGlobal(token)
1588 self.error("token %s %s unexpected at the top level" % (
1589 token[0], token[1]))
1590 token = self.parseGlobal(token)