Searched defs:suffix_indicator (Results 1 – 8 of 8) sorted by relevance
| /third_party/mindspore/tests/ut/python/dataset/ |
| D | test_text_bert_tokenizer.py | 172 vocab_list, suffix_indicator='##', argument
201 vocab_list, suffix_indicator='##', argument
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/kernels/ |
| D | bert_tokenizer_op.h | 42 … : wordpiece_tokenizer_(vocab, suffix_indicator, max_bytes_per_token, unknown_token, with_offsets), in wordpiece_tokenizer_() argument
|
| D | wordpiece_tokenizer_op.cc | 29 …rOp::WordpieceTokenizerOp(const std::shared_ptr<Vocab> &vocab, const std::string &suffix_indicator, in WordpieceTokenizerOp()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/include/dataset/ |
| D | text.h | 98 …: BertTokenizer(vocab, StringToChar(suffix_indicator), max_bytes_per_token, StringToChar(unknown_t… in BertTokenizer()
537 …: WordpieceTokenizer(vocab, StringToChar(suffix_indicator), max_bytes_per_token, StringToChar(unkn… in WordpieceTokenizer()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/ir/ |
| D | bindings.cc | 48 bool with_offsets) { in __anone8b21dd50302()
238 … int32_t max_bytes_per_token, const std::string &unknown_token, bool with_offsets) { in __anone8b21dd52402()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/ir/kernels/ |
| D | text_ir.cc | 86 …n::BertTokenizerOperation(const std::shared_ptr<Vocab> &vocab, const std::string &suffix_indicator, in BertTokenizerOperation()
475 const std::string &suffix_indicator, in WordpieceTokenizerOperation()
|
| /third_party/mindspore/mindspore/dataset/text/ |
| D | transforms.py | 513 def __init__(self, vocab, suffix_indicator='##', max_bytes_per_token=100, argument
691 … def __init__(self, vocab, suffix_indicator='##', max_bytes_per_token=100, unknown_token='[UNK]', argument
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/api/ |
| D | text.cc | 93 …izer::BertTokenizer(const std::shared_ptr<Vocab> &vocab, const std::vector<char> &suffix_indicator, in BertTokenizer()
404 …:WordpieceTokenizer(const std::shared_ptr<Vocab> &vocab, const std::vector<char> &suffix_indicator, in WordpieceTokenizer()
|