Searched refs:SPieceTokenizerLoadType (Results 1 – 10 of 10) sorted by relevance
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/kernels/ |
| D | sentence_piece_tokenizer_op.h | 37 …SentencePieceTokenizerOp(const std::shared_ptr<SentencePieceVocab> vocab, SPieceTokenizerLoadType …
41 … const SPieceTokenizerLoadType load_type, const SPieceTokenizerOutType out_type);
59 SPieceTokenizerLoadType load_type_;
|
| D | sentence_piece_tokenizer_op.cc | 28 const SPieceTokenizerLoadType load_type, in SentencePieceTokenizerOp()
41 const SPieceTokenizerLoadType load_type, in SentencePieceTokenizerOp()
|
| /third_party/mindspore/mindspore/dataset/text/ |
| D | transforms.py | 50 from .utils import JiebaMode, NormalizeForm, to_str, SPieceTokenizerOutType, SPieceTokenizerLoadType
77 SPieceTokenizerLoadType.FILE: cde.SPieceTokenizerLoadType.DE_SPIECE_TOKENIZER_LOAD_KFILE,
78 SPieceTokenizerLoadType.MODEL: cde.SPieceTokenizerLoadType.DE_SPIECE_TOKENIZER_LOAD_KMODEL
|
| D | __init__.py | 31 SPieceTokenizerOutType, SPieceTokenizerLoadType
|
| D | utils.py | 375 class SPieceTokenizerLoadType(IntEnum): class
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/ir/ |
| D | bindings.cc | 254 PYBIND_REGISTER(SPieceTokenizerLoadType, 0, ([](const py::module *m) { in __anonf0f064542702()
255 … (void)py::enum_<SPieceTokenizerLoadType>(*m, "SPieceTokenizerLoadType", py::arithmetic()) in __anonf0f064542702()
256 .value("DE_SPIECE_TOKENIZER_LOAD_KFILE", SPieceTokenizerLoadType::kFile) in __anonf0f064542702()
257 .value("DE_SPIECE_TOKENIZER_LOAD_KMODEL", SPieceTokenizerLoadType::kModel) in __anonf0f064542702()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/ir/kernels/ |
| D | text_ir.cc | 336 …: vocab_(vocab), vocab_path_(std::string()), load_type_(SPieceTokenizerLoadType::kModel), out_type… in SentencePieceTokenizerOperation()
340 …: vocab_(nullptr), vocab_path_(vocab_path), load_type_(SPieceTokenizerLoadType::kFile), out_type_(… in SentencePieceTokenizerOperation()
348 if (load_type_ == SPieceTokenizerLoadType::kModel) { in ValidateParams()
374 if (load_type_ == SPieceTokenizerLoadType::kModel) { in Build()
|
| D | text_ir.h | 255 SPieceTokenizerLoadType load_type_;
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/include/dataset/ |
| D | constants.h | 145 enum class SPieceTokenizerLoadType { enum
|
| /third_party/mindspore/tests/ut/cpp/dataset/ |
| D | sentence_piece_vocab_op_test.cc | 165 …new SentencePieceTokenizerOp(spm, SPieceTokenizerLoadType::kModel, SPieceTokenizerOutType::kString… in TEST_F()
|