Searched defs:indexedToken (Results 1 – 2 of 2) sorted by relevance
34 type indexedToken struct { struct35 Line int // line position of this token in the source36 ID tokenID // identifier of the text in the dictionary
91 func generateHashes(h hash, q int, toks []indexedToken, dict *dictionary) ([]uint32, tokenRanges) {