Searched defs:suffix_indicator (Results 1 – 8 of 8) sorted by relevance
| /third_party/mindspore/mindspore-src/source/tests/ut/python/dataset/ |
| D | test_text_bert_tokenizer.py | 172 vocab_list, suffix_indicator='##', argument
201 vocab_list, suffix_indicator='##', argument
|
| /third_party/mindspore/mindspore-src/source/mindspore/ccsrc/minddata/dataset/text/kernels/ |
| D | bert_tokenizer_op.h | 42 … : wordpiece_tokenizer_(vocab, suffix_indicator, max_bytes_per_token, unknown_token, with_offsets), in wordpiece_tokenizer_() argument
|
| D | wordpiece_tokenizer_op.cc | 29 …rOp::WordpieceTokenizerOp(const std::shared_ptr<Vocab> &vocab, const std::string &suffix_indicator, in WordpieceTokenizerOp()
|
| /third_party/mindspore/mindspore-src/source/mindspore/ccsrc/minddata/dataset/include/dataset/ |
| D | text.h | 345 …: BertTokenizer(vocab, StringToChar(suffix_indicator), max_bytes_per_token, StringToChar(unknown_t… in BertTokenizer()
1010 …: WordpieceTokenizer(vocab, StringToChar(suffix_indicator), max_bytes_per_token, StringToChar(unkn… in WordpieceTokenizer()
|
| /third_party/mindspore/mindspore-src/source/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/ir/ |
| D | bindings.cc | 58 bool with_offsets) { in __anon2bada04b0502()
281 … int32_t max_bytes_per_token, const std::string &unknown_token, bool with_offsets) { in __anon2bada04b2c02()
|
| /third_party/mindspore/mindspore-src/source/mindspore/python/mindspore/dataset/text/ |
| D | transforms.py | 1054 def __init__(self, vocab, suffix_indicator='##', max_bytes_per_token=100, unknown_token='[UNK]', argument
1271 … def __init__(self, vocab, suffix_indicator='##', max_bytes_per_token=100, unknown_token='[UNK]', argument
|
| /third_party/mindspore/mindspore-src/source/mindspore/ccsrc/minddata/dataset/text/ir/kernels/ |
| D | text_ir.cc | 120 …n::BertTokenizerOperation(const std::shared_ptr<Vocab> &vocab, const std::string &suffix_indicator, in BertTokenizerOperation()
648 const std::string &suffix_indicator, in WordpieceTokenizerOperation()
|
| /third_party/mindspore/mindspore-src/source/mindspore/ccsrc/minddata/dataset/api/ |
| D | text.cc | 103 …izer::BertTokenizer(const std::shared_ptr<Vocab> &vocab, const std::vector<char> &suffix_indicator, in BertTokenizer()
447 …:WordpieceTokenizer(const std::shared_ptr<Vocab> &vocab, const std::vector<char> &suffix_indicator, in WordpieceTokenizer()
|