Searched refs:TokenizedStringEntry (Results 1 – 7 of 7) sorted by relevance
| /external/pigweed/pw_tokenizer/py/ |
| D | tokens_test.py | 98 def _entries(*strings: str) -> Iterator[tokens.TokenizedStringEntry]:
100 yield tokens.TokenizedStringEntry(default_hash(string), string)
203 tokens.TokenizedStringEntry(
205 tokens.TokenizedStringEntry(
217 tokens.TokenizedStringEntry(3, 'three'),
218 tokens.TokenizedStringEntry(
230 tokens.TokenizedStringEntry(5, 'five'),
242 tokens.TokenizedStringEntry(
244 tokens.TokenizedStringEntry(
258 tokens.TokenizedStringEntry(4, 'four'),
[all …]
|
| D | detokenize_test.py | 106 tokens.TokenizedStringEntry(0xcdab,
116 tokens.TokenizedStringEntry(1,
133 tokens.TokenizedStringEntry(2,
148 tokens.TokenizedStringEntry(2,
165 tokens.TokenizedStringEntry(1,
317 tokens.TokenizedStringEntry(
319 tokens.TokenizedStringEntry(token, 'newer'),
320 tokens.TokenizedStringEntry(
322 tokens.TokenizedStringEntry(
324 tokens.TokenizedStringEntry(token, 'C: %s'),
[all …]
|
| /external/pigweed/pw_tokenizer/py/pw_tokenizer/ |
| D | tokens.py | 76 class TokenizedStringEntry: class
116 def __init__(self, entries: Iterable[TokenizedStringEntry] = ()): argument
119 self._database: Dict[_EntryKey, TokenizedStringEntry] = {
125 self._cache: Optional[Dict[int, List[TokenizedStringEntry]]] = None
134 return cls((TokenizedStringEntry(tokenize(string), string, domain)
145 def token_to_entries(self) -> Dict[int, List[TokenizedStringEntry]]:
154 def entries(self) -> ValuesView[TokenizedStringEntry]:
158 def collisions(self) -> Iterator[Tuple[int, List[TokenizedStringEntry]]]:
166 all_entries: Iterable[TokenizedStringEntry], argument
168 ) -> List[TokenizedStringEntry]:
[all …]
|
| D | database.py | 70 domain: Pattern[str]) -> Iterator[tokens.TokenizedStringEntry]:
85 entry = tokens.TokenizedStringEntry(
|
| D | detokenize.py | 167 entry: tokens.TokenizedStringEntry
|
| /external/pigweed/pw_tokenizer/public/pw_tokenizer/ |
| D | detokenize.h | 40 using TokenizedStringEntry = std::pair<FormatString, uint32_t /*date removed*/>; variable
47 const std::span<const TokenizedStringEntry>& entries,
95 std::unordered_map<uint32_t, std::vector<TokenizedStringEntry>> database_;
|
| /external/pigweed/pw_tokenizer/ |
| D | detokenize.cc | 72 const std::span<const TokenizedStringEntry>& entries, in DetokenizedString()
120 ? std::span<TokenizedStringEntry>() in Detokenize()
|