1// Copyright 2017 The Wuffs Authors. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// https://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package token 16 17// MaxIntBits is the largest size (in bits) of the i8, u8, i16, u16, etc. 18// integer types. 19const MaxIntBits = 64 20 21// ID is a token type. Every identifier (in the programming language sense), 22// keyword, operator and literal has its own ID. 23// 24// Some IDs are built-in: the "func" keyword always has the same numerical ID 25// value. Others are mapped at runtime. For example, the ID value for the 26// "foobar" identifier (e.g. a variable name) is looked up in a Map. 27type ID uint32 28 29// Str returns a string form of x. 30func (x ID) Str(m *Map) string { return m.ByID(x) } 31 32func (x ID) AmbiguousForm() ID { 33 if x >= ID(len(ambiguousForms)) { 34 return 0 35 } 36 return ambiguousForms[x] 37} 38 39func (x ID) UnaryForm() ID { 40 if x >= ID(len(unaryForms)) { 41 return 0 42 } 43 return unaryForms[x] 44} 45 46func (x ID) BinaryForm() ID { 47 if x >= ID(len(binaryForms)) { 48 return 0 49 } 50 return binaryForms[x] 51} 52 53func (x ID) AssociativeForm() ID { 54 if x >= ID(len(associativeForms)) { 55 return 0 56 } 57 return associativeForms[x] 58} 59 60func (x ID) IsBuiltIn() bool { return x < nBuiltInIDs } 61 62func (x ID) IsUnaryOp() bool { return minOp <= x && x <= maxOp && unaryForms[x] != 0 } 63func (x ID) IsBinaryOp() bool { return minOp <= x && x <= maxOp && binaryForms[x] != 0 } 64func (x ID) IsAssociativeOp() bool { return minOp <= x && x <= maxOp && associativeForms[x] != 0 } 65 66func (x ID) IsLiteral(m *Map) bool { 67 if x < nBuiltInIDs { 68 return minBuiltInLiteral <= x && x <= maxBuiltInLiteral 69 } else if s := m.ByID(x); s != "" { 70 return !alpha(s[0]) 71 } 72 return false 73} 74 75func (x ID) IsNumLiteral(m *Map) bool { 76 if x < nBuiltInIDs { 77 return minBuiltInNumLiteral <= x && x <= maxBuiltInNumLiteral 78 } else if s := m.ByID(x); s != "" { 79 return numeric(s[0]) 80 } 81 return false 82} 83 84func (x ID) IsStrLiteral(m *Map) bool { 85 if x < nBuiltInIDs { 86 return false 87 } else if s := m.ByID(x); s != "" { 88 return s[0] == '"' 89 } 90 return false 91} 92 93func (x ID) IsIdent(m *Map) bool { 94 if x < nBuiltInIDs { 95 return minBuiltInIdent <= x && x <= maxBuiltInIdent 96 } else if s := m.ByID(x); s != "" { 97 return alpha(s[0]) 98 } 99 return false 100} 101 102func (x ID) IsTightLeft() bool { return x < ID(len(isTightLeft)) && isTightLeft[x] } 103func (x ID) IsTightRight() bool { return x < ID(len(isTightRight)) && isTightRight[x] } 104 105func (x ID) IsAssign() bool { return minAssign <= x && x <= maxAssign } 106func (x ID) IsCannotAssignTo() bool { return minCannotAssignTo <= x && x <= maxCannotAssignTo } 107func (x ID) IsClose() bool { return minClose <= x && x <= maxClose } 108func (x ID) IsKeyword() bool { return minKeyword <= x && x <= maxKeyword } 109func (x ID) IsNumType() bool { return minNumType <= x && x <= maxNumType } 110func (x ID) IsNumTypeOrIdeal() bool { return minNumTypeOrIdeal <= x && x <= maxNumTypeOrIdeal } 111func (x ID) IsOpen() bool { return minOpen <= x && x <= maxOpen } 112 113func (x ID) IsImplicitSemicolon(m *Map) bool { 114 return x.IsClose() || x.IsKeyword() || x.IsIdent(m) || x.IsLiteral(m) 115} 116 117func (x ID) IsXOp() bool { return minXOp <= x && x <= maxXOp } 118func (x ID) IsXUnaryOp() bool { return minXOp <= x && x <= maxXOp && unaryForms[x] != 0 } 119func (x ID) IsXBinaryOp() bool { return minXOp <= x && x <= maxXOp && binaryForms[x] != 0 } 120func (x ID) IsXAssociativeOp() bool { return minXOp <= x && x <= maxXOp && associativeForms[x] != 0 } 121 122func (x ID) SmallPowerOf2Value() int { 123 switch x { 124 case ID1: 125 return 1 126 case ID2: 127 return 2 128 case ID4: 129 return 4 130 case ID8: 131 return 8 132 case ID16: 133 return 16 134 case ID32: 135 return 32 136 case ID64: 137 return 64 138 case ID128: 139 return 128 140 case ID256: 141 return 256 142 } 143 return 0 144} 145 146// QID is a qualified ID, such as "foo.bar". QID[0] is "foo"'s ID and QID[1] is 147// "bar"'s. QID[0] may be 0 for a plain "bar". 148type QID [2]ID 149 150func (x QID) IsZero() bool { return x == QID{} } 151 152// Str returns a string form of x. 153func (x QID) Str(m *Map) string { 154 if x[0] != 0 { 155 return m.ByID(x[0]) + "." + m.ByID(x[1]) 156 } 157 return m.ByID(x[1]) 158} 159 160// QQID is a double-qualified ID, such as "receiverPkg.receiverName.funcName". 161type QQID [3]ID 162 163func (x QQID) IsZero() bool { return x == QQID{} } 164 165// Str returns a string form of x. 166func (x QQID) Str(m *Map) string { 167 if x[0] != 0 { 168 return m.ByID(x[0]) + "." + m.ByID(x[1]) + "." + m.ByID(x[2]) 169 } 170 if x[1] != 0 { 171 return m.ByID(x[1]) + "." + m.ByID(x[2]) 172 } 173 return m.ByID(x[2]) 174} 175 176// Token combines an ID and the line number it was seen. 177type Token struct { 178 ID ID 179 Line uint32 180} 181 182// nBuiltInIDs is the number of built-in IDs. The packing is: 183// - Zero is invalid. 184// - [ 0x01, 0x0F] are squiggly punctuation, such as "(", ")" and ";". 185// - [ 0x10, 0x2F] are squiggly assignments, such as "=" and "+=". 186// - [ 0x30, 0x4F] are operators, such as "+", "==" and "not". 187// - [ 0x50, 0x7F] are x-ops (disambiguation forms): unary vs binary "+". 188// - [ 0x80, 0x9F] are keywords, such as "if" and "return". 189// - [ 0xA0, 0xAF] are type modifiers, such as "ptr" and "slice". 190// - [ 0xB0, 0xBF] are literals, such as "false" and "true". 191// - [ 0xC0, 0xFF] are reserved. 192// - [0x100, 0x3FF] are identifiers, such as "bool", "u32" and "read_u8". 193// 194// "Squiggly" means a sequence of non-alpha-numeric characters, such as "+" and 195// "&=". Roughly speaking, their IDs range in [0x01, 0x4F], or disambiguation 196// forms range in [0x50, 0x7F], but vice versa does not necessarily hold. For 197// example, the "and" operator is not "squiggly" but it is within [0x01, 0x4F]. 198const ( 199 nBuiltInSymbolicIDs = ID(0x80) // 128 200 nBuiltInIDs = ID(0x400) // 1024 201) 202 203const ( 204 IDInvalid = ID(0) 205 206 minOpen = 0x02 207 maxOpen = 0x04 208 209 IDOpenParen = ID(0x02) 210 IDOpenBracket = ID(0x03) 211 IDOpenCurly = ID(0x04) 212 213 minClose = 0x05 214 maxClose = 0x07 215 216 IDCloseParen = ID(0x05) 217 IDCloseBracket = ID(0x06) 218 IDCloseCurly = ID(0x07) 219 220 IDDot = ID(0x08) 221 IDDotDot = ID(0x09) 222 IDComma = ID(0x0A) 223 IDExclam = ID(0x0B) 224 IDQuestion = ID(0x0C) 225 IDColon = ID(0x0D) 226 IDSemicolon = ID(0x0E) 227 IDDollar = ID(0x0F) 228) 229 230const ( 231 minAssign = 0x10 232 maxAssign = 0x2F 233 234 IDPlusEq = ID(0x10) 235 IDMinusEq = ID(0x11) 236 IDStarEq = ID(0x12) 237 IDSlashEq = ID(0x13) 238 IDShiftLEq = ID(0x14) 239 IDShiftREq = ID(0x15) 240 IDAmpEq = ID(0x16) 241 IDPipeEq = ID(0x17) 242 IDHatEq = ID(0x18) 243 IDPercentEq = ID(0x19) 244 IDTildeModShiftLEq = ID(0x1A) 245 IDTildeModPlusEq = ID(0x1B) 246 IDTildeModMinusEq = ID(0x1C) 247 IDTildeSatPlusEq = ID(0x1D) 248 IDTildeSatMinusEq = ID(0x1E) 249 250 IDEq = ID(0x20) 251 IDEqQuestion = ID(0x21) 252) 253 254const ( 255 minOp = 0x30 256 minAmbiguousOp = 0x30 257 maxAmbiguousOp = 0x4F 258 minXOp = 0x50 259 maxXOp = 0x7F 260 maxOp = 0x7F 261 262 IDPlus = ID(0x30) 263 IDMinus = ID(0x31) 264 IDStar = ID(0x32) 265 IDSlash = ID(0x33) 266 IDShiftL = ID(0x34) 267 IDShiftR = ID(0x35) 268 IDAmp = ID(0x36) 269 IDPipe = ID(0x37) 270 IDHat = ID(0x38) 271 IDPercent = ID(0x39) 272 IDTildeModShiftL = ID(0x3A) 273 IDTildeModPlus = ID(0x3B) 274 IDTildeModMinus = ID(0x3C) 275 IDTildeSatPlus = ID(0x3D) 276 IDTildeSatMinus = ID(0x3E) 277 278 IDNotEq = ID(0x40) 279 IDLessThan = ID(0x41) 280 IDLessEq = ID(0x42) 281 IDEqEq = ID(0x43) 282 IDGreaterEq = ID(0x44) 283 IDGreaterThan = ID(0x45) 284 285 IDAnd = ID(0x48) 286 IDOr = ID(0x49) 287 IDNot = ID(0x4A) 288 IDAs = ID(0x4B) 289 290 // TODO: are these unused? Can we drop them (and their XUnary forms)? 291 IDRef = ID(0x4C) 292 IDDeref = ID(0x4D) 293 294 // The IDXFoo IDs are not returned by the tokenizer. They are used by the 295 // ast.Node ID-typed fields to disambiguate e.g. unary vs binary plus. 296 297 IDXUnaryPlus = ID(0x50) 298 IDXUnaryMinus = ID(0x51) 299 IDXUnaryNot = ID(0x52) 300 IDXUnaryRef = ID(0x53) 301 IDXUnaryDeref = ID(0x54) 302 303 IDXBinaryPlus = ID(0x58) 304 IDXBinaryMinus = ID(0x59) 305 IDXBinaryStar = ID(0x5A) 306 IDXBinarySlash = ID(0x5B) 307 IDXBinaryShiftL = ID(0x5C) 308 IDXBinaryShiftR = ID(0x5D) 309 IDXBinaryAmp = ID(0x5E) 310 IDXBinaryPipe = ID(0x5F) 311 IDXBinaryHat = ID(0x60) 312 IDXBinaryPercent = ID(0x61) 313 IDXBinaryTildeModShiftL = ID(0x62) 314 IDXBinaryTildeModPlus = ID(0x63) 315 IDXBinaryTildeModMinus = ID(0x64) 316 IDXBinaryTildeSatPlus = ID(0x65) 317 IDXBinaryTildeSatMinus = ID(0x66) 318 IDXBinaryNotEq = ID(0x67) 319 IDXBinaryLessThan = ID(0x68) 320 IDXBinaryLessEq = ID(0x69) 321 IDXBinaryEqEq = ID(0x6A) 322 IDXBinaryGreaterEq = ID(0x6B) 323 IDXBinaryGreaterThan = ID(0x6C) 324 IDXBinaryAnd = ID(0x6D) 325 IDXBinaryOr = ID(0x6E) 326 IDXBinaryAs = ID(0x6F) 327 328 IDXAssociativePlus = ID(0x70) 329 IDXAssociativeStar = ID(0x71) 330 IDXAssociativeAmp = ID(0x72) 331 IDXAssociativePipe = ID(0x73) 332 IDXAssociativeHat = ID(0x74) 333 IDXAssociativeAnd = ID(0x75) 334 IDXAssociativeOr = ID(0x76) 335) 336 337const ( 338 minKeyword = 0x80 339 maxKeyword = 0x9F 340 341 // TODO: sort these by name, when the list has stabilized. 342 IDFunc = ID(0x80) 343 IDAssert = ID(0x81) 344 IDWhile = ID(0x82) 345 IDIf = ID(0x83) 346 IDElse = ID(0x84) 347 IDReturn = ID(0x85) 348 IDBreak = ID(0x86) 349 IDContinue = ID(0x87) 350 IDStruct = ID(0x88) 351 IDUse = ID(0x89) 352 IDVar = ID(0x8A) 353 IDPre = ID(0x8B) 354 IDInv = ID(0x8C) 355 IDPost = ID(0x8D) 356 IDVia = ID(0x8E) 357 IDPub = ID(0x8F) 358 IDPri = ID(0x90) 359 IDConst = ID(0x91) 360 IDIterate = ID(0x92) 361 IDYield = ID(0x93) 362 IDIOBind = ID(0x94) 363 IDIOLimit = ID(0x95) 364) 365 366const ( 367 minTypeModifier = 0xA0 368 maxTypeModifier = 0xAF 369 370 IDArray = ID(0xA0) 371 IDNptr = ID(0xA1) 372 IDPtr = ID(0xA2) 373 IDSlice = ID(0xA3) 374 IDTable = ID(0xA4) 375) 376 377const ( 378 minBuiltInLiteral = 0xB0 379 minBuiltInNumLiteral = 0xC0 380 maxBuiltInNumLiteral = 0xCF 381 maxBuiltInLiteral = 0xCF 382 383 IDFalse = ID(0xB0) 384 IDTrue = ID(0xB1) 385 IDNothing = ID(0xB2) 386 IDNullptr = ID(0xB3) 387 IDOk = ID(0xB4) 388 389 ID0 = ID(0xC0) 390 ID1 = ID(0xC1) 391 ID2 = ID(0xC2) 392 ID4 = ID(0xC3) 393 ID8 = ID(0xC4) 394 ID16 = ID(0xC5) 395 ID32 = ID(0xC6) 396 ID64 = ID(0xC7) 397 ID128 = ID(0xC8) 398 ID256 = ID(0xC9) 399) 400 401const ( 402 minBuiltInIdent = 0x100 403 minCannotAssignTo = 0x100 404 maxCannotAssignTo = 0x102 405 minNumTypeOrIdeal = 0x10F 406 minNumType = 0x110 407 maxNumType = 0x117 408 maxNumTypeOrIdeal = 0x117 409 maxBuiltInIdent = 0x3FF 410 411 // -------- 0x100 block. 412 413 IDArgs = ID(0x100) 414 IDCoroutineResumed = ID(0x101) 415 IDThis = ID(0x102) 416 417 IDT1 = ID(0x108) 418 IDT2 = ID(0x109) 419 IDDagger1 = ID(0x10A) 420 IDDagger2 = ID(0x10B) 421 422 IDQNullptr = ID(0x10C) 423 IDQPlaceholder = ID(0x10D) 424 IDQTypeExpr = ID(0x10E) 425 426 // It is important that IDQIdeal is right next to the IDI8..IDU64 block. 427 // See the ID.IsNumTypeOrIdeal method. 428 IDQIdeal = ID(0x10F) 429 430 IDI8 = ID(0x110) 431 IDI16 = ID(0x111) 432 IDI32 = ID(0x112) 433 IDI64 = ID(0x113) 434 IDU8 = ID(0x114) 435 IDU16 = ID(0x115) 436 IDU32 = ID(0x116) 437 IDU64 = ID(0x117) 438 439 IDBase = ID(0x120) 440 IDBool = ID(0x121) 441 IDEmptyStruct = ID(0x122) 442 IDIOReader = ID(0x123) 443 IDIOWriter = ID(0x124) 444 IDStatus = ID(0x125) 445 IDUtility = ID(0x126) 446 447 IDRangeIEU32 = ID(0x130) 448 IDRangeIIU32 = ID(0x131) 449 IDRangeIEU64 = ID(0x132) 450 IDRangeIIU64 = ID(0x133) 451 IDRectIEU32 = ID(0x134) 452 IDRectIIU32 = ID(0x135) 453 454 IDFrameConfig = ID(0x150) 455 IDImageConfig = ID(0x151) 456 IDPixelBuffer = ID(0x152) 457 IDPixelConfig = ID(0x153) 458 IDPixelSwizzler = ID(0x154) 459 460 IDDecodeFrameOptions = ID(0x158) 461 462 IDCanUndoByte = ID(0x160) 463 IDCountSince = ID(0x161) 464 IDHistoryAvailable = ID(0x162) 465 IDMark = ID(0x163) 466 IDPosition = ID(0x164) 467 IDSince = ID(0x165) 468 IDSkip = ID(0x166) 469 IDSkipFast = ID(0x167) 470 IDTake = ID(0x168) 471 472 IDCopyFromSlice = ID(0x170) 473 IDCopyNFromHistory = ID(0x171) 474 IDCopyNFromHistoryFast = ID(0x172) 475 IDCopyNFromReader = ID(0x173) 476 IDCopyNFromSlice = ID(0x174) 477 478 // -------- 0x180 block. 479 480 IDUndoByte = ID(0x180) 481 IDReadU8 = ID(0x181) 482 483 IDReadU16BE = ID(0x182) 484 IDReadU16LE = ID(0x183) 485 486 IDReadU8AsU32 = ID(0x189) 487 IDReadU16BEAsU32 = ID(0x18A) 488 IDReadU16LEAsU32 = ID(0x18B) 489 IDReadU24BEAsU32 = ID(0x18C) 490 IDReadU24LEAsU32 = ID(0x18D) 491 IDReadU32BE = ID(0x18E) 492 IDReadU32LE = ID(0x18F) 493 494 IDReadU8AsU64 = ID(0x191) 495 IDReadU16BEAsU64 = ID(0x192) 496 IDReadU16LEAsU64 = ID(0x193) 497 IDReadU24BEAsU64 = ID(0x194) 498 IDReadU24LEAsU64 = ID(0x195) 499 IDReadU32BEAsU64 = ID(0x196) 500 IDReadU32LEAsU64 = ID(0x197) 501 IDReadU40BEAsU64 = ID(0x198) 502 IDReadU40LEAsU64 = ID(0x199) 503 IDReadU48BEAsU64 = ID(0x19A) 504 IDReadU48LEAsU64 = ID(0x19B) 505 IDReadU56BEAsU64 = ID(0x19C) 506 IDReadU56LEAsU64 = ID(0x19D) 507 IDReadU64BE = ID(0x19E) 508 IDReadU64LE = ID(0x19F) 509 510 // -------- 511 512 IDPeekU8 = ID(0x1A1) 513 514 IDPeekU16BE = ID(0x1A2) 515 IDPeekU16LE = ID(0x1A3) 516 517 IDPeekU8AsU32 = ID(0x1A9) 518 IDPeekU16BEAsU32 = ID(0x1AA) 519 IDPeekU16LEAsU32 = ID(0x1AB) 520 IDPeekU24BEAsU32 = ID(0x1AC) 521 IDPeekU24LEAsU32 = ID(0x1AD) 522 IDPeekU32BE = ID(0x1AE) 523 IDPeekU32LE = ID(0x1AF) 524 525 IDPeekU8AsU64 = ID(0x1B1) 526 IDPeekU16BEAsU64 = ID(0x1B2) 527 IDPeekU16LEAsU64 = ID(0x1B3) 528 IDPeekU24BEAsU64 = ID(0x1B4) 529 IDPeekU24LEAsU64 = ID(0x1B5) 530 IDPeekU32BEAsU64 = ID(0x1B6) 531 IDPeekU32LEAsU64 = ID(0x1B7) 532 IDPeekU40BEAsU64 = ID(0x1B8) 533 IDPeekU40LEAsU64 = ID(0x1B9) 534 IDPeekU48BEAsU64 = ID(0x1BA) 535 IDPeekU48LEAsU64 = ID(0x1BB) 536 IDPeekU56BEAsU64 = ID(0x1BC) 537 IDPeekU56LEAsU64 = ID(0x1BD) 538 IDPeekU64BE = ID(0x1BE) 539 IDPeekU64LE = ID(0x1BF) 540 541 // -------- 542 543 // TODO: IDUnwriteU8? 544 545 IDWriteU8 = ID(0x1C1) 546 IDWriteU16BE = ID(0x1C2) 547 IDWriteU16LE = ID(0x1C3) 548 IDWriteU24BE = ID(0x1C4) 549 IDWriteU24LE = ID(0x1C5) 550 IDWriteU32BE = ID(0x1C6) 551 IDWriteU32LE = ID(0x1C7) 552 IDWriteU40BE = ID(0x1C8) 553 IDWriteU40LE = ID(0x1C9) 554 IDWriteU48BE = ID(0x1CA) 555 IDWriteU48LE = ID(0x1CB) 556 IDWriteU56BE = ID(0x1CC) 557 IDWriteU56LE = ID(0x1CD) 558 IDWriteU64BE = ID(0x1CE) 559 IDWriteU64LE = ID(0x1CF) 560 561 // -------- 562 563 IDWriteFastU8 = ID(0x1E1) 564 IDWriteFastU16BE = ID(0x1E2) 565 IDWriteFastU16LE = ID(0x1E3) 566 IDWriteFastU24BE = ID(0x1E4) 567 IDWriteFastU24LE = ID(0x1E5) 568 IDWriteFastU32BE = ID(0x1E6) 569 IDWriteFastU32LE = ID(0x1E7) 570 IDWriteFastU40BE = ID(0x1E8) 571 IDWriteFastU40LE = ID(0x1E9) 572 IDWriteFastU48BE = ID(0x1EA) 573 IDWriteFastU48LE = ID(0x1EB) 574 IDWriteFastU56BE = ID(0x1EC) 575 IDWriteFastU56LE = ID(0x1ED) 576 IDWriteFastU64BE = ID(0x1EE) 577 IDWriteFastU64LE = ID(0x1EF) 578 579 // -------- 0x200 block. 580 581 IDInitialize = ID(0x200) 582 IDReset = ID(0x201) 583 IDSet = ID(0x202) 584 IDUnroll = ID(0x203) 585 IDUpdate = ID(0x204) 586 587 // TODO: range/rect methods like intersection and contains? 588 589 IDHighBits = ID(0x220) 590 IDLowBits = ID(0x221) 591 IDMax = ID(0x222) 592 IDMin = ID(0x223) 593 594 IDIsError = ID(0x230) 595 IDIsOK = ID(0x231) 596 IDIsSuspension = ID(0x232) 597 598 IDAvailable = ID(0x240) 599 IDHeight = ID(0x241) 600 IDLength = ID(0x242) 601 IDPrefix = ID(0x243) 602 IDRow = ID(0x244) 603 IDStride = ID(0x245) 604 IDSuffix = ID(0x246) 605 IDWidth = ID(0x247) 606 IDIO = ID(0x248) 607 IDLimit = ID(0x249) 608 IDData = ID(0x24A) 609) 610 611var builtInsByID = [nBuiltInIDs]string{ 612 IDOpenParen: "(", 613 IDCloseParen: ")", 614 IDOpenBracket: "[", 615 IDCloseBracket: "]", 616 IDOpenCurly: "{", 617 IDCloseCurly: "}", 618 619 IDDot: ".", 620 IDDotDot: "..", 621 IDComma: ",", 622 IDExclam: "!", 623 IDQuestion: "?", 624 IDColon: ":", 625 IDSemicolon: ";", 626 IDDollar: "$", 627 628 IDPlusEq: "+=", 629 IDMinusEq: "-=", 630 IDStarEq: "*=", 631 IDSlashEq: "/=", 632 IDShiftLEq: "<<=", 633 IDShiftREq: ">>=", 634 IDAmpEq: "&=", 635 IDPipeEq: "|=", 636 IDHatEq: "^=", 637 IDPercentEq: "%=", 638 IDTildeModShiftLEq: "~mod<<=", 639 IDTildeModPlusEq: "~mod+=", 640 IDTildeModMinusEq: "~mod-=", 641 IDTildeSatPlusEq: "~sat+=", 642 IDTildeSatMinusEq: "~sat-=", 643 644 IDEq: "=", 645 IDEqQuestion: "=?", 646 647 IDPlus: "+", 648 IDMinus: "-", 649 IDStar: "*", 650 IDSlash: "/", 651 IDShiftL: "<<", 652 IDShiftR: ">>", 653 IDAmp: "&", 654 IDPipe: "|", 655 IDHat: "^", 656 IDPercent: "%", 657 IDTildeModShiftL: "~mod<<", 658 IDTildeModPlus: "~mod+", 659 IDTildeModMinus: "~mod-", 660 IDTildeSatPlus: "~sat+", 661 IDTildeSatMinus: "~sat-", 662 663 IDNotEq: "<>", 664 IDLessThan: "<", 665 IDLessEq: "<=", 666 IDEqEq: "==", 667 IDGreaterEq: ">=", 668 IDGreaterThan: ">", 669 670 IDAnd: "and", 671 IDOr: "or", 672 IDNot: "not", 673 IDAs: "as", 674 IDRef: "ref", 675 IDDeref: "deref", 676 677 IDFunc: "func", 678 IDAssert: "assert", 679 IDWhile: "while", 680 IDIf: "if", 681 IDElse: "else", 682 IDReturn: "return", 683 IDBreak: "break", 684 IDContinue: "continue", 685 IDStruct: "struct", 686 IDUse: "use", 687 IDVar: "var", 688 IDPre: "pre", 689 IDInv: "inv", 690 IDPost: "post", 691 IDVia: "via", 692 IDPub: "pub", 693 IDPri: "pri", 694 IDConst: "const", 695 IDIterate: "iterate", 696 IDYield: "yield", 697 IDIOBind: "io_bind", 698 IDIOLimit: "io_limit", 699 700 IDArray: "array", 701 IDNptr: "nptr", 702 IDPtr: "ptr", 703 IDSlice: "slice", 704 IDTable: "table", 705 706 IDFalse: "false", 707 IDTrue: "true", 708 IDNothing: "nothing", 709 IDNullptr: "nullptr", 710 IDOk: "ok", 711 712 ID0: "0", 713 ID1: "1", 714 ID2: "2", 715 ID4: "4", 716 ID8: "8", 717 ID16: "16", 718 ID32: "32", 719 ID64: "64", 720 ID128: "128", 721 ID256: "256", 722 723 // -------- 0x100 block. 724 725 IDArgs: "args", 726 IDCoroutineResumed: "coroutine_resumed", 727 IDThis: "this", 728 729 // Some of the next few IDs are never returned by the tokenizer, as it 730 // rejects non-ASCII input. The string representations "¶", "ℤ" etc. are 731 // specifically non-ASCII so that no user-defined (non built-in) identifier 732 // will conflict with them. 733 734 // IDDaggerN is used by the type checker as a dummy-valued built-in ID to 735 // represent a generic type. 736 IDT1: "T1", 737 IDT2: "T2", 738 IDDagger1: "†", // U+2020 DAGGER 739 IDDagger2: "‡", // U+2021 DOUBLE DAGGER 740 741 // IDQNullptr is used by the type checker to build an artificial MType for 742 // the nullptr literal. 743 IDQNullptr: "«Nullptr»", 744 745 // IDQPlaceholder is used by the type checker to build an artificial MType 746 // for AST nodes that aren't expression nodes or type expression nodes, 747 // such as struct definition nodes and statement nodes. Its presence means 748 // that the node is type checked. 749 IDQPlaceholder: "«Placeholder»", 750 751 // IDQTypeExpr is used by the type checker to build an artificial MType for 752 // type expression AST nodes. 753 IDQTypeExpr: "«TypeExpr»", 754 755 // IDQIdeal is used by the type checker to build an artificial MType for 756 // ideal integers (in mathematical terms, the integer ring ℤ), as opposed 757 // to a realized integer type whose range is restricted. For example, the 758 // base.u16 type is restricted to [0x0000, 0xFFFF]. 759 IDQIdeal: "«Ideal»", 760 761 // Change MaxIntBits if a future update adds an i128 or u128 type. 762 IDI8: "i8", 763 IDI16: "i16", 764 IDI32: "i32", 765 IDI64: "i64", 766 IDU8: "u8", 767 IDU16: "u16", 768 IDU32: "u32", 769 IDU64: "u64", 770 771 IDBase: "base", 772 IDBool: "bool", 773 IDEmptyStruct: "empty_struct", 774 IDIOReader: "io_reader", 775 IDIOWriter: "io_writer", 776 IDStatus: "status", 777 IDUtility: "utility", 778 779 IDRangeIEU32: "range_ie_u32", 780 IDRangeIIU32: "range_ii_u32", 781 IDRangeIEU64: "range_ie_u64", 782 IDRangeIIU64: "range_ii_u64", 783 IDRectIEU32: "rect_ie_u32", 784 IDRectIIU32: "rect_ii_u32", 785 786 IDFrameConfig: "frame_config", 787 IDImageConfig: "image_config", 788 IDPixelBuffer: "pixel_buffer", 789 IDPixelConfig: "pixel_config", 790 IDPixelSwizzler: "pixel_swizzler", 791 792 IDDecodeFrameOptions: "decode_frame_options", 793 794 IDCanUndoByte: "can_undo_byte", 795 IDCountSince: "count_since", 796 IDHistoryAvailable: "history_available", 797 IDMark: "mark", 798 IDPosition: "position", 799 IDSince: "since", 800 IDSkip: "skip", 801 IDSkipFast: "skip_fast", 802 IDTake: "take", 803 804 IDCopyFromSlice: "copy_from_slice", 805 IDCopyNFromHistory: "copy_n_from_history", 806 IDCopyNFromHistoryFast: "copy_n_from_history_fast", 807 IDCopyNFromReader: "copy_n_from_reader", 808 IDCopyNFromSlice: "copy_n_from_slice", 809 810 // -------- 0x180 block. 811 812 IDUndoByte: "undo_byte", 813 IDReadU8: "read_u8", 814 815 IDReadU16BE: "read_u16be", 816 IDReadU16LE: "read_u16le", 817 818 IDReadU8AsU32: "read_u8_as_u32", 819 IDReadU16BEAsU32: "read_u16be_as_u32", 820 IDReadU16LEAsU32: "read_u16le_as_u32", 821 IDReadU24BEAsU32: "read_u24be_as_u32", 822 IDReadU24LEAsU32: "read_u24le_as_u32", 823 IDReadU32BE: "read_u32be", 824 IDReadU32LE: "read_u32le", 825 826 IDReadU8AsU64: "read_u8_as_u64", 827 IDReadU16BEAsU64: "read_u16be_as_u64", 828 IDReadU16LEAsU64: "read_u16le_as_u64", 829 IDReadU24BEAsU64: "read_u24be_as_u64", 830 IDReadU24LEAsU64: "read_u24le_as_u64", 831 IDReadU32BEAsU64: "read_u32be_as_u64", 832 IDReadU32LEAsU64: "read_u32le_as_u64", 833 IDReadU40BEAsU64: "read_u40be_as_u64", 834 IDReadU40LEAsU64: "read_u40le_as_u64", 835 IDReadU48BEAsU64: "read_u48be_as_u64", 836 IDReadU48LEAsU64: "read_u48le_as_u64", 837 IDReadU56BEAsU64: "read_u56be_as_u64", 838 IDReadU56LEAsU64: "read_u56le_as_u64", 839 IDReadU64BE: "read_u64be", 840 IDReadU64LE: "read_u64le", 841 842 // -------- 843 844 IDPeekU8: "peek_u8", 845 846 IDPeekU16BE: "peek_u16be", 847 IDPeekU16LE: "peek_u16le", 848 849 IDPeekU8AsU32: "peek_u8_as_u32", 850 IDPeekU16BEAsU32: "peek_u16be_as_u32", 851 IDPeekU16LEAsU32: "peek_u16le_as_u32", 852 IDPeekU24BEAsU32: "peek_u24be_as_u32", 853 IDPeekU24LEAsU32: "peek_u24le_as_u32", 854 IDPeekU32BE: "peek_u32be", 855 IDPeekU32LE: "peek_u32le", 856 857 IDPeekU8AsU64: "peek_u8_as_u64", 858 IDPeekU16BEAsU64: "peek_u16be_as_u64", 859 IDPeekU16LEAsU64: "peek_u16le_as_u64", 860 IDPeekU24BEAsU64: "peek_u24be_as_u64", 861 IDPeekU24LEAsU64: "peek_u24le_as_u64", 862 IDPeekU32BEAsU64: "peek_u32be_as_u64", 863 IDPeekU32LEAsU64: "peek_u32le_as_u64", 864 IDPeekU40BEAsU64: "peek_u40be_as_u64", 865 IDPeekU40LEAsU64: "peek_u40le_as_u64", 866 IDPeekU48BEAsU64: "peek_u48be_as_u64", 867 IDPeekU48LEAsU64: "peek_u48le_as_u64", 868 IDPeekU56BEAsU64: "peek_u56be_as_u64", 869 IDPeekU56LEAsU64: "peek_u56le_as_u64", 870 IDPeekU64BE: "peek_u64be", 871 IDPeekU64LE: "peek_u64le", 872 873 // -------- 874 875 IDWriteU8: "write_u8", 876 IDWriteU16BE: "write_u16be", 877 IDWriteU16LE: "write_u16le", 878 IDWriteU24BE: "write_u24be", 879 IDWriteU24LE: "write_u24le", 880 IDWriteU32BE: "write_u32be", 881 IDWriteU32LE: "write_u32le", 882 IDWriteU40BE: "write_u40be", 883 IDWriteU40LE: "write_u40le", 884 IDWriteU48BE: "write_u48be", 885 IDWriteU48LE: "write_u48le", 886 IDWriteU56BE: "write_u56be", 887 IDWriteU56LE: "write_u56le", 888 IDWriteU64BE: "write_u64be", 889 IDWriteU64LE: "write_u64le", 890 891 // -------- 892 893 IDWriteFastU8: "write_fast_u8", 894 IDWriteFastU16BE: "write_fast_u16be", 895 IDWriteFastU16LE: "write_fast_u16le", 896 IDWriteFastU24BE: "write_fast_u24be", 897 IDWriteFastU24LE: "write_fast_u24le", 898 IDWriteFastU32BE: "write_fast_u32be", 899 IDWriteFastU32LE: "write_fast_u32le", 900 IDWriteFastU40BE: "write_fast_u40be", 901 IDWriteFastU40LE: "write_fast_u40le", 902 IDWriteFastU48BE: "write_fast_u48be", 903 IDWriteFastU48LE: "write_fast_u48le", 904 IDWriteFastU56BE: "write_fast_u56be", 905 IDWriteFastU56LE: "write_fast_u56le", 906 IDWriteFastU64BE: "write_fast_u64be", 907 IDWriteFastU64LE: "write_fast_u64le", 908 909 // -------- 0x200 block. 910 911 IDInitialize: "initialize", 912 IDReset: "reset", 913 IDSet: "set", 914 IDUnroll: "unroll", 915 IDUpdate: "update", 916 917 IDHighBits: "high_bits", 918 IDLowBits: "low_bits", 919 IDMax: "max", 920 IDMin: "min", 921 922 IDIsError: "is_error", 923 IDIsOK: "is_ok", 924 IDIsSuspension: "is_suspension", 925 926 IDAvailable: "available", 927 IDHeight: "height", 928 IDLength: "length", 929 IDPrefix: "prefix", 930 IDRow: "row", 931 IDStride: "stride", 932 IDSuffix: "suffix", 933 IDWidth: "width", 934 IDIO: "io", 935 IDLimit: "limit", 936 IDData: "data", 937} 938 939var builtInsByName = map[string]ID{} 940 941func init() { 942 for i, name := range builtInsByID { 943 if name != "" { 944 builtInsByName[name] = ID(i) 945 } 946 } 947} 948 949// squiggles are built-in IDs that aren't alpha-numeric. 950var squiggles = [256]ID{ 951 '(': IDOpenParen, 952 ')': IDCloseParen, 953 '[': IDOpenBracket, 954 ']': IDCloseBracket, 955 '{': IDOpenCurly, 956 '}': IDCloseCurly, 957 958 ',': IDComma, 959 '!': IDExclam, 960 '?': IDQuestion, 961 ':': IDColon, 962 ';': IDSemicolon, 963 '$': IDDollar, 964} 965 966type suffixLexer struct { 967 suffix string 968 id ID 969} 970 971// lexers lex ambiguous 1-byte squiggles. For example, "&" might be the start 972// of "&^" or "&=". 973// 974// The order of the []suffixLexer elements matters. The first match wins. Since 975// we want to lex greedily, longer suffixes should be earlier in the slice. 976var lexers = [256][]suffixLexer{ 977 '.': { 978 {".", IDDotDot}, 979 {"", IDDot}, 980 }, 981 '&': { 982 {"=", IDAmpEq}, 983 {"", IDAmp}, 984 }, 985 '|': { 986 {"=", IDPipeEq}, 987 {"", IDPipe}, 988 }, 989 '^': { 990 {"=", IDHatEq}, 991 {"", IDHat}, 992 }, 993 '+': { 994 {"=", IDPlusEq}, 995 {"", IDPlus}, 996 }, 997 '-': { 998 {"=", IDMinusEq}, 999 {"", IDMinus}, 1000 }, 1001 '*': { 1002 {"=", IDStarEq}, 1003 {"", IDStar}, 1004 }, 1005 '/': { 1006 {"=", IDSlashEq}, 1007 {"", IDSlash}, 1008 }, 1009 '%': { 1010 {"=", IDPercentEq}, 1011 {"", IDPercent}, 1012 }, 1013 '=': { 1014 {"=", IDEqEq}, 1015 {"?", IDEqQuestion}, 1016 {"", IDEq}, 1017 }, 1018 '<': { 1019 {"<=", IDShiftLEq}, 1020 {"<", IDShiftL}, 1021 {"=", IDLessEq}, 1022 {">", IDNotEq}, 1023 {"", IDLessThan}, 1024 }, 1025 '>': { 1026 {">=", IDShiftREq}, 1027 {">", IDShiftR}, 1028 {"=", IDGreaterEq}, 1029 {"", IDGreaterThan}, 1030 }, 1031 '~': { 1032 {"mod<<=", IDTildeModShiftLEq}, 1033 {"mod<<", IDTildeModShiftL}, 1034 {"mod+=", IDTildeModPlusEq}, 1035 {"mod+", IDTildeModPlus}, 1036 {"mod-=", IDTildeModMinusEq}, 1037 {"mod-", IDTildeModMinus}, 1038 {"sat+=", IDTildeSatPlusEq}, 1039 {"sat+", IDTildeSatPlus}, 1040 {"sat-=", IDTildeSatMinusEq}, 1041 {"sat-", IDTildeSatMinus}, 1042 }, 1043} 1044 1045var ambiguousForms = [nBuiltInSymbolicIDs]ID{ 1046 IDXUnaryPlus: IDPlus, 1047 IDXUnaryMinus: IDMinus, 1048 IDXUnaryNot: IDNot, 1049 IDXUnaryRef: IDRef, 1050 IDXUnaryDeref: IDDeref, 1051 1052 IDXBinaryPlus: IDPlus, 1053 IDXBinaryMinus: IDMinus, 1054 IDXBinaryStar: IDStar, 1055 IDXBinarySlash: IDSlash, 1056 IDXBinaryShiftL: IDShiftL, 1057 IDXBinaryShiftR: IDShiftR, 1058 IDXBinaryAmp: IDAmp, 1059 IDXBinaryPipe: IDPipe, 1060 IDXBinaryHat: IDHat, 1061 IDXBinaryPercent: IDPercent, 1062 IDXBinaryTildeModShiftL: IDTildeModShiftL, 1063 IDXBinaryTildeModPlus: IDTildeModPlus, 1064 IDXBinaryTildeModMinus: IDTildeModMinus, 1065 IDXBinaryTildeSatPlus: IDTildeSatPlus, 1066 IDXBinaryTildeSatMinus: IDTildeSatMinus, 1067 IDXBinaryNotEq: IDNotEq, 1068 IDXBinaryLessThan: IDLessThan, 1069 IDXBinaryLessEq: IDLessEq, 1070 IDXBinaryEqEq: IDEqEq, 1071 IDXBinaryGreaterEq: IDGreaterEq, 1072 IDXBinaryGreaterThan: IDGreaterThan, 1073 IDXBinaryAnd: IDAnd, 1074 IDXBinaryOr: IDOr, 1075 IDXBinaryAs: IDAs, 1076 1077 IDXAssociativePlus: IDPlus, 1078 IDXAssociativeStar: IDStar, 1079 IDXAssociativeAmp: IDAmp, 1080 IDXAssociativePipe: IDPipe, 1081 IDXAssociativeHat: IDHat, 1082 IDXAssociativeAnd: IDAnd, 1083 IDXAssociativeOr: IDOr, 1084} 1085 1086func init() { 1087 addXForms(&unaryForms) 1088 addXForms(&binaryForms) 1089 addXForms(&associativeForms) 1090} 1091 1092// addXForms modifies table so that, if table[x] == y, then table[y] = y. 1093// 1094// For example, for the unaryForms table, the explicit entries are like: 1095// IDPlus: IDXUnaryPlus, 1096// and this function implicitly addes entries like: 1097// IDXUnaryPlus: IDXUnaryPlus, 1098func addXForms(table *[nBuiltInSymbolicIDs]ID) { 1099 implicitEntries := [nBuiltInSymbolicIDs]bool{} 1100 for _, y := range table { 1101 if y != 0 { 1102 implicitEntries[y] = true 1103 } 1104 } 1105 for y, implicit := range implicitEntries { 1106 if implicit { 1107 table[y] = ID(y) 1108 } 1109 } 1110} 1111 1112var unaryForms = [nBuiltInSymbolicIDs]ID{ 1113 IDPlus: IDXUnaryPlus, 1114 IDMinus: IDXUnaryMinus, 1115 IDNot: IDXUnaryNot, 1116 IDRef: IDXUnaryRef, 1117 IDDeref: IDXUnaryDeref, 1118} 1119 1120var binaryForms = [nBuiltInSymbolicIDs]ID{ 1121 IDPlusEq: IDXBinaryPlus, 1122 IDMinusEq: IDXBinaryMinus, 1123 IDStarEq: IDXBinaryStar, 1124 IDSlashEq: IDXBinarySlash, 1125 IDShiftLEq: IDXBinaryShiftL, 1126 IDShiftREq: IDXBinaryShiftR, 1127 IDAmpEq: IDXBinaryAmp, 1128 IDPipeEq: IDXBinaryPipe, 1129 IDHatEq: IDXBinaryHat, 1130 IDPercentEq: IDXBinaryPercent, 1131 IDTildeModShiftLEq: IDXBinaryTildeModShiftL, 1132 IDTildeModPlusEq: IDXBinaryTildeModPlus, 1133 IDTildeModMinusEq: IDXBinaryTildeModMinus, 1134 IDTildeSatPlusEq: IDXBinaryTildeSatPlus, 1135 IDTildeSatMinusEq: IDXBinaryTildeSatMinus, 1136 1137 IDPlus: IDXBinaryPlus, 1138 IDMinus: IDXBinaryMinus, 1139 IDStar: IDXBinaryStar, 1140 IDSlash: IDXBinarySlash, 1141 IDShiftL: IDXBinaryShiftL, 1142 IDShiftR: IDXBinaryShiftR, 1143 IDAmp: IDXBinaryAmp, 1144 IDPipe: IDXBinaryPipe, 1145 IDHat: IDXBinaryHat, 1146 IDPercent: IDXBinaryPercent, 1147 IDTildeModShiftL: IDXBinaryTildeModShiftL, 1148 IDTildeModPlus: IDXBinaryTildeModPlus, 1149 IDTildeModMinus: IDXBinaryTildeModMinus, 1150 IDTildeSatPlus: IDXBinaryTildeSatPlus, 1151 IDTildeSatMinus: IDXBinaryTildeSatMinus, 1152 1153 IDNotEq: IDXBinaryNotEq, 1154 IDLessThan: IDXBinaryLessThan, 1155 IDLessEq: IDXBinaryLessEq, 1156 IDEqEq: IDXBinaryEqEq, 1157 IDGreaterEq: IDXBinaryGreaterEq, 1158 IDGreaterThan: IDXBinaryGreaterThan, 1159 IDAnd: IDXBinaryAnd, 1160 IDOr: IDXBinaryOr, 1161 IDAs: IDXBinaryAs, 1162} 1163 1164var associativeForms = [nBuiltInSymbolicIDs]ID{ 1165 IDPlus: IDXAssociativePlus, 1166 IDStar: IDXAssociativeStar, 1167 IDAmp: IDXAssociativeAmp, 1168 IDPipe: IDXAssociativePipe, 1169 IDHat: IDXAssociativeHat, 1170 // TODO: IDTildeModPlus, IDTildeSatPlus? 1171 IDAnd: IDXAssociativeAnd, 1172 IDOr: IDXAssociativeOr, 1173} 1174 1175var isTightLeft = [...]bool{ 1176 IDCloseParen: true, 1177 IDOpenBracket: true, 1178 IDCloseBracket: true, 1179 1180 IDDot: true, 1181 IDDotDot: true, 1182 IDComma: true, 1183 IDExclam: true, 1184 IDQuestion: true, 1185 IDColon: true, 1186 IDSemicolon: true, 1187} 1188 1189var isTightRight = [...]bool{ 1190 IDOpenParen: true, 1191 IDOpenBracket: true, 1192 1193 IDDot: true, 1194 IDDotDot: true, 1195 IDExclam: true, 1196 IDQuestion: true, 1197 IDColon: true, 1198 IDDollar: true, 1199} 1200