Searched refs:Detokenizer (Results 1 – 11 of 11) sorted by relevance
104 detok = detokenize.Detokenizer(114 detok = detokenize.Detokenizer(131 detok = detokenize.Detokenizer(147 detok = detokenize.Detokenizer(tokens.Database([163 detok = detokenize.Detokenizer(177 detok = detokenize.Detokenizer(io.BytesIO(EMPTY_ELF))189 detok = detokenize.Detokenizer(io.BytesIO(EMPTY_ELF), show_errors=True)199 detok = detokenize.Detokenizer(io.BytesIO(EMPTY_ELF), show_errors=True)214 detok = detokenize.Detokenizer(io.BytesIO(EMPTY_ELF))231 detok = detokenize.Detokenizer(io.BytesIO(ELF_WITH_TOKENIZER_SECTIONS))[all …]
34 Detokenizer* HandleToPointer(jlong handle) { in HandleToPointer()35 Detokenizer* detokenizer = nullptr; in HandleToPointer()41 jlong PointerToHandle(Detokenizer* detokenizer) { in PointerToHandle()62 PointerToHandle(new Detokenizer(tokens.ok() ? tokens : TokenDatabase())); in DETOKENIZER_METHOD()
29 public class Detokenizer { class42 public Detokenizer() { in Detokenizer() method in Detokenizer46 public Detokenizer(byte[] tokenDatabase) { in Detokenizer() method in Detokenizer
100 Detokenizer::Detokenizer(const TokenDatabase& database) { in Detokenizer() function in pw::tokenizer::Detokenizer106 DetokenizedString Detokenizer::Detokenize( in Detokenize()
47 Detokenizer detok_;136 Detokenizer detok_;232 Detokenizer detok_;
63 static Detokenizer detokenizer(TokenDatabase::Create<kBasicData>()); in LLVMFuzzerTestOneInput()
776 To detokenize in Python, import ``Detokenizer`` from the ``pw_tokenizer``783 detokenizer = pw_tokenizer.Detokenizer('path/to/database.csv', 'other/path.elf')790 class, which can be used in place of the standard ``Detokenizer``. This class807 Detokenizer detokenizer(TokenDatabase::Create(token_database_array));823 Detokenizer OpenDatabase(std::string_view path) {832 return Detokenizer(database);834 return Detokenizer(kDefaultDatabase);1004 ``pw_tokenizer.Detokenizer``.
75 class Detokenizer {79 Detokenizer(const TokenDatabase& database);
171 class Detokenizer: class246 self._detokenizer = Detokenizer(*(path.load() for path in self.paths))255 self._detokenizer = Detokenizer(*(path.load()261 _Detokenizer = Union[Detokenizer, AutoUpdatingDetokenizer]459 detokenizer = Detokenizer(tokens.Database.merged(*databases),
16 from pw_tokenizer.detokenize import AutoUpdatingDetokenizer, Detokenizer
72 detokenizer = detokenize.Detokenizer(tokens.Database.merged(*databases),