Lines Matching full:token
406 def push(self, token): argument
407 self.tokens.insert(0, token);
410 print("Last token: ", self.last)
411 print("Token queue: ", self.tokens)
414 def token(self): member in CLexer
633 def error(self, msg, token=-1): argument
638 if token != -1:
639 print("Got token ", token)
643 def debug(self, msg, token=-1): argument
645 if token != -1:
646 print("Got token ", token)
677 def parseComment(self, token): argument
679 self.top_comment = token[1]
680 if self.comment == None or token[1][0] == '*':
681 self.comment = token[1];
683 self.comment = self.comment + token[1]
684 token = self.lexer.token()
692 return token
923 def parsePreproc(self, token): argument
925 print("=> preproc ", token, self.lexer.tokens)
926 name = token[1]
928 token = self.lexer.token()
929 if token == None:
931 if token[0] == 'preproc':
932 self.index_add(token[1], self.filename, not self.is_header,
934 return self.lexer.token()
935 return token
937 token = self.lexer.token()
938 if token == None:
940 if token[0] == 'preproc':
942 name = token[1]
944 token = self.lexer.token()
945 while token != None and token[0] == 'preproc' and \
946 token[1][0] != '#':
947 lst.append(token[1])
948 token = self.lexer.token()
956 return token
1012 token = self.lexer.token()
1013 while token != None and token[0] == 'preproc' and \
1014 token[1][0] != '#':
1015 token = self.lexer.token()
1016 return token
1019 # token acquisition on top of the lexer, it handle internally
1023 def token(self): member in CParser
1026 token = self.lexer.token()
1027 while token != None:
1028 if token[0] == 'comment':
1029 token = self.parseComment(token)
1031 elif token[0] == 'preproc':
1032 token = self.parsePreproc(token)
1034 elif token[0] == "name" and token[1] == "__const":
1035 token = ("name", "const")
1036 return token
1037 elif token[0] == "name" and token[1] == "__attribute":
1038 token = self.lexer.token()
1039 while token != None and token[1] != ";":
1040 token = self.lexer.token()
1041 return token
1042 elif token[0] == "name" and token[1] in ignored_words:
1043 (n, info) = ignored_words[token[1]]
1046 token = self.lexer.token()
1048 token = self.lexer.token()
1052 print("=> ", token)
1053 return token
1059 def parseTypedef(self, token): argument
1060 if token == None:
1062 token = self.parseType(token)
1063 if token == None:
1068 #self.debug("end typedef type", token)
1069 while token != None:
1070 if token[0] == "name":
1071 name = token[1]
1089 token = self.token()
1092 return token
1093 #self.debug("end typedef", token)
1094 if token != None and token[0] == 'sep' and token[1] == ',':
1096 token = self.token()
1097 while token != None and token[0] == "op":
1098 type = type + token[1]
1099 token = self.token()
1100 elif token != None and token[0] == 'sep' and token[1] == ';':
1102 elif token != None and token[0] == 'name':
1106 self.error("parsing typedef: expecting ';'", token)
1107 return token
1108 token = self.token()
1109 return token
1115 def parseBlock(self, token): argument
1116 while token != None:
1117 if token[0] == "sep" and token[1] == "{":
1118 token = self.token()
1119 token = self.parseBlock(token)
1120 elif token[0] == "sep" and token[1] == "}":
1122 token = self.token()
1123 return token
1126 oldtok = token
1127 token = self.token()
1129 if token[0] == "sep" and token[1] == "(":
1132 token = self.token()
1133 elif token[0] == "name":
1134 token = self.token()
1135 if token[0] == "sep" and (token[1] == ";" or
1136 token[1] == "," or token[1] == "="):
1147 token = self.token()
1148 return token
1153 def parseStruct(self, token): argument
1155 #self.debug("start parseStruct", token)
1156 while token != None:
1157 if token[0] == "sep" and token[1] == "{":
1158 token = self.token()
1159 token = self.parseTypeBlock(token)
1160 elif token[0] == "sep" and token[1] == "}":
1162 #self.debug("end parseStruct", token)
1164 token = self.token()
1165 return token
1168 #self.debug("before parseType", token)
1169 token = self.parseType(token)
1170 #self.debug("after parseType", token)
1171 if token != None and token[0] == "name":
1172 fname = token[1]
1173 token = self.token()
1174 if token[0] == "sep" and token[1] == ";":
1176 token = self.token()
1180 self.error("parseStruct: expecting ;", token)
1181 elif token != None and token[0] == "sep" and token[1] == "{":
1182 token = self.token()
1183 token = self.parseTypeBlock(token)
1184 if token != None and token[0] == "name":
1185 token = self.token()
1186 if token != None and token[0] == "sep" and token[1] == ";":
1187 token = self.token()
1189 self.error("parseStruct: expecting ;", token)
1191 self.error("parseStruct: name", token)
1192 token = self.token()
1195 #self.debug("end parseStruct", token)
1197 return token
1202 def parseEnumBlock(self, token): argument
1208 while token != None:
1209 if token[0] == "sep" and token[1] == "{":
1210 token = self.token()
1211 token = self.parseTypeBlock(token)
1212 elif token[0] == "sep" and token[1] == "}":
1218 token = self.token()
1219 return token
1220 elif token[0] == "name":
1226 name = token[1]
1228 token = self.token()
1229 if token[0] == "op" and token[1][0] == "=":
1231 if len(token[1]) > 1:
1232 value = token[1][1:]
1233 token = self.token()
1234 while token[0] != "sep" or (token[1] != ',' and
1235 token[1] != '}'):
1236 value = value + token[1]
1237 token = self.token()
1244 if token[0] == "sep" and token[1] == ",":
1245 token = self.token()
1247 token = self.token()
1248 return token
1254 def parseTypeBlock(self, token): argument
1255 while token != None:
1256 if token[0] == "sep" and token[1] == "{":
1257 token = self.token()
1258 token = self.parseTypeBlock(token)
1259 elif token[0] == "sep" and token[1] == "}":
1260 token = self.token()
1261 return token
1263 token = self.token()
1264 return token
1269 # if inside, the name token is pushed back before returning
1271 def parseType(self, token): argument
1275 if token == None:
1276 return token
1278 while token[0] == "name" and (
1279 token[1] == "const" or \
1280 token[1] == "unsigned" or \
1281 token[1] == "signed"):
1283 self.type = token[1]
1285 self.type = self.type + " " + token[1]
1286 token = self.token()
1288 if token[0] == "name" and (token[1] == "long" or token[1] == "short"):
1290 self.type = token[1]
1292 self.type = self.type + " " + token[1]
1293 if token[0] == "name" and token[1] == "int":
1299 elif token[0] == "name" and token[1] == "struct":
1301 self.type = token[1]
1303 self.type = self.type + " " + token[1]
1304 token = self.token()
1306 if token[0] == "name":
1307 nametok = token
1308 token = self.token()
1309 if token != None and token[0] == "sep" and token[1] == "{":
1310 token = self.token()
1311 token = self.parseStruct(token)
1312 elif token != None and token[0] == "op" and token[1] == "*":
1314 token = self.token()
1315 while token != None and token[0] == "op" and token[1] == "*":
1317 token = self.token()
1318 if token[0] == "name":
1319 nametok = token
1320 token = self.token()
1322 self.error("struct : expecting name", token)
1323 return token
1324 elif token != None and token[0] == "name" and nametok != None:
1326 return token
1329 self.lexer.push(token)
1330 token = nametok
1331 return token
1333 elif token[0] == "name" and token[1] == "enum":
1335 self.type = token[1]
1337 self.type = self.type + " " + token[1]
1339 token = self.token()
1340 if token != None and token[0] == "sep" and token[1] == "{":
1341 token = self.token()
1342 token = self.parseEnumBlock(token)
1344 self.error("parsing enum: expecting '{'", token)
1346 if token != None and token[0] != "name":
1347 self.lexer.push(token)
1348 token = ("name", "enum")
1350 enum_type = token[1]
1355 return token
1357 elif token[0] == "name":
1359 self.type = token[1]
1361 self.type = self.type + " " + token[1]
1364 token)
1365 return token
1366 token = self.token()
1367 while token != None and (token[0] == "op" or
1368 token[0] == "name" and token[1] == "const"):
1369 self.type = self.type + " " + token[1]
1370 token = self.token()
1375 if token != None and token[0] == "sep" and token[1] == '(':
1376 self.type = self.type + token[1]
1377 token = self.token()
1378 while token != None and token[0] == "op" and token[1] == '*':
1379 self.type = self.type + token[1]
1380 token = self.token()
1381 if token == None or token[0] != "name" :
1382 self.error("parsing function type, name expected", token);
1383 return token
1384 self.type = self.type + token[1]
1385 nametok = token
1386 token = self.token()
1387 if token != None and token[0] == "sep" and token[1] == ')':
1388 self.type = self.type + token[1]
1389 token = self.token()
1390 if token != None and token[0] == "sep" and token[1] == '(':
1391 token = self.token()
1393 token = self.parseSignature(token);
1396 self.error("parsing function type, '(' expected", token);
1397 return token
1399 self.error("parsing function type, ')' expected", token);
1400 return token
1401 self.lexer.push(token)
1402 token = nametok
1403 return token
1408 if token != None and token[0] == "name":
1409 nametok = token
1410 token = self.token()
1411 if token != None and token[0] == "sep" and token[1] == '[':
1413 while token != None and token[0] == "sep" and token[1] == '[':
1414 self.type = self.type + token[1]
1415 token = self.token()
1416 while token != None and token[0] != 'sep' and \
1417 token[1] != ']' and token[1] != ';':
1418 self.type = self.type + token[1]
1419 token = self.token()
1420 if token != None and token[0] == 'sep' and token[1] == ']':
1421 self.type = self.type + token[1]
1422 token = self.token()
1424 self.error("parsing array type, ']' expected", token);
1425 return token
1426 elif token != None and token[0] == "sep" and token[1] == ':':
1428 token = self.token()
1429 token = self.token()
1430 self.lexer.push(token)
1431 token = nametok
1433 return token
1438 def parseSignature(self, token): argument
1440 if token != None and token[0] == "sep" and token[1] == ')':
1442 token = self.token()
1443 return token
1444 while token != None:
1445 token = self.parseType(token)
1446 if token != None and token[0] == "name":
1447 signature.append((self.type, token[1], None))
1448 token = self.token()
1449 elif token != None and token[0] == "sep" and token[1] == ',':
1450 token = self.token()
1452 elif token != None and token[0] == "sep" and token[1] == ')':
1458 if token != None and token[0] == "sep":
1459 if token[1] == ',':
1460 token = self.token()
1462 elif token[1] == ')':
1463 token = self.token()
1466 return token
1472 def parseGlobal(self, token): argument
1474 if token[1] == 'extern':
1475 token = self.token()
1476 if token == None:
1477 return token
1478 if token[0] == 'string':
1479 if token[1] == 'C':
1480 token = self.token()
1481 if token == None:
1482 return token
1483 if token[0] == 'sep' and token[1] == "{":
1484 token = self.token()
1486 while token != None and (token[0] != 'sep' or
1487 token[1] != "}"):
1488 if token[0] == 'name':
1489 token = self.parseGlobal(token)
1492 "token %s %s unexpected at the top level" % (
1493 token[0], token[1]))
1494 token = self.parseGlobal(token)
1496 token = self.token()
1497 return token
1499 return token
1500 elif token[1] == 'static':
1502 token = self.token()
1503 if token == None or token[0] != 'name':
1504 return token
1506 if token[1] == 'typedef':
1507 token = self.token()
1508 return self.parseTypedef(token)
1510 token = self.parseType(token)
1512 if token == None or token[0] != "name":
1513 return token
1515 self.name = token[1]
1516 token = self.token()
1517 while token != None and (token[0] == "sep" or token[0] == "op"):
1518 if token[0] == "sep":
1519 if token[1] == "[":
1520 type = type + token[1]
1521 token = self.token()
1522 while token != None and (token[0] != "sep" or \
1523 token[1] != ";"):
1524 type = type + token[1]
1525 token = self.token()
1527 if token != None and token[0] == "op" and token[1] == "=":
1531 token = self.token()
1532 if token[0] == 'sep' and token[1] == '{':
1533 token = self.token()
1534 token = self.parseBlock(token)
1537 while token != None and (token[0] != "sep" or \
1538 (token[1] != ';' and token[1] != ',')):
1539 token = self.token()
1541 if token == None or token[0] != "sep" or (token[1] != ';' and
1542 token[1] != ','):
1545 if token != None and token[0] == "sep":
1546 if token[1] == ";":
1548 token = self.token()
1556 elif token[1] == "(":
1557 token = self.token()
1558 token = self.parseSignature(token)
1559 if token == None:
1561 if token[0] == "sep" and token[1] == ";":
1566 token = self.token()
1567 elif token[0] == "sep" and token[1] == "{":
1572 token = self.token()
1573 token = self.parseBlock(token);
1574 elif token[1] == ',':
1579 token = self.token()
1580 while token != None and token[0] == "sep":
1581 type = type + token[1]
1582 token = self.token()
1583 if token != None and token[0] == "name":
1584 self.name = token[1]
1585 token = self.token()
1589 return token
1593 token = self.token()
1594 while token != None:
1595 if token[0] == 'name':
1596 token = self.parseGlobal(token)
1598 self.error("token %s %s unexpected at the top level" % (
1599 token[0], token[1]))
1600 token = self.parseGlobal(token)
1641 for token in tokens:
1643 c = token[0]
1646 elif len(token) < 3:
1649 lower = token.lower()
1653 elif token in self.xref:
1654 self.xref[token].append(id)
1656 self.xref[token] = [id]
2027 for token in tokens:
2028 if tok == token:
2030 tok = token
2031 output.write(" <ref name='%s'/>\n" % (token))