Home
last modified time | relevance | path

Searched refs:SPieceTokenizerOutType (Results 1 – 15 of 15) sorted by relevance

/third_party/mindspore/tests/ut/python/dataset/
Dtest_sentencepiece_tokenizer.py19 from mindspore.dataset.text import SentencePieceModel, to_str, SPieceTokenizerOutType
27 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
34 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
46 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
58 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
71 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
83 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.INT)
96 tokenizer = text.SentencePieceTokenizer("./m.model", out_type=SPieceTokenizerOutType.STRING)
109 tokenizer = text.SentencePieceTokenizer("./m.model", out_type=SPieceTokenizerOutType.INT)
122 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
[all …]
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/kernels/
Dsentence_piece_tokenizer_op.h38 const SPieceTokenizerOutType out_type);
41 … const SPieceTokenizerLoadType load_type, const SPieceTokenizerOutType out_type);
56 SPieceTokenizerOutType out_type_;
Dsentence_piece_tokenizer_op.cc29 const SPieceTokenizerOutType out_type) in SentencePieceTokenizerOp()
42 const SPieceTokenizerOutType out_type) in SentencePieceTokenizerOp()
70 if (out_type_ == SPieceTokenizerOutType::kString) { in Compute()
/third_party/mindspore/tests/ut/cpp/dataset/
Dc_api_text_sentence_piece_vocab_test.cc57 …e_shared<text::SentencePieceTokenizer>(vocab, mindspore::dataset::SPieceTokenizerOutType::kString); in TEST_F()
120 …ed<text::SentencePieceTokenizer>(vocab_model, mindspore::dataset::SPieceTokenizerOutType::kString); in TEST_F()
199 …ed<text::SentencePieceTokenizer>(vocab_model, mindspore::dataset::SPieceTokenizerOutType::kString); in TEST_F()
223 …ed<text::SentencePieceTokenizer>(vocab_model, mindspore::dataset::SPieceTokenizerOutType::kString); in TEST_F()
247 …ed<text::SentencePieceTokenizer>(vocab_model, mindspore::dataset::SPieceTokenizerOutType::kString); in TEST_F()
271 …d<text::SentencePieceTokenizer>(vocab_model4, mindspore::dataset::SPieceTokenizerOutType::kString); in TEST_F()
Dsentence_piece_vocab_op_test.cc165 …new SentencePieceTokenizerOp(spm, SPieceTokenizerLoadType::kModel, SPieceTokenizerOutType::kString… in TEST_F()
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/ir/
Dbindings.cc178 ….def(py::init([](const std::shared_ptr<SentencePieceVocab> &vocab, SPieceTokenizerOutType out_type… in __anonf0f064541902()
183 .def(py::init([](const std::string &vocab_path, SPieceTokenizerOutType out_type) { in __anonf0f064541902()
261 PYBIND_REGISTER(SPieceTokenizerOutType, 0, ([](const py::module *m) { in __anonf0f064542802()
262 … (void)py::enum_<SPieceTokenizerOutType>(*m, "SPieceTokenizerOutType", py::arithmetic()) in __anonf0f064542802()
263 .value("DE_SPIECE_TOKENIZER_OUTTYPE_KString", SPieceTokenizerOutType::kString) in __anonf0f064542802()
264 .value("DE_SPIECE_TOKENIZER_OUTTYPE_KINT", SPieceTokenizerOutType::kInt) in __anonf0f064542802()
/third_party/mindspore/mindspore/dataset/text/
Dtransforms.py50 from .utils import JiebaMode, NormalizeForm, to_str, SPieceTokenizerOutType, SPieceTokenizerLoadType
82 SPieceTokenizerOutType.STRING: cde.SPieceTokenizerOutType.DE_SPIECE_TOKENIZER_OUTTYPE_KString,
83 SPieceTokenizerOutType.INT: cde.SPieceTokenizerOutType.DE_SPIECE_TOKENIZER_OUTTYPE_KINT
D__init__.py31 SPieceTokenizerOutType, SPieceTokenizerLoadType
Dutils.py362 class SPieceTokenizerOutType(IntEnum): class
Dvalidators.py524 from .utils import SPieceTokenizerOutType
530 …type_check(out_type, (SPieceTokenizerOutType,), "out_type is not an instance of SPieceTokenizerOut…
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/api/
Dtext.cc316 Data(const std::shared_ptr<SentencePieceVocab> &vocab, SPieceTokenizerOutType out_type) in Data()
318 Data(const std::vector<char> &vocab_path, SPieceTokenizerOutType out_type) in Data()
322 SPieceTokenizerOutType out_type_;
326 SPieceTokenizerOutType out_type) in SentencePieceTokenizer()
329 …nizer::SentencePieceTokenizer(const std::vector<char> &vocab_path, SPieceTokenizerOutType out_type) in SentencePieceTokenizer()
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/ir/kernels/
Dtext_ir.h240 …enizerOperation(const std::shared_ptr<SentencePieceVocab> &vocab, SPieceTokenizerOutType out_type);
242 SentencePieceTokenizerOperation(const std::string &vocab_path, SPieceTokenizerOutType out_type);
256 SPieceTokenizerOutType out_type_;
Dtext_ir.cc335 SPieceTokenizerOutType out_type) in SentencePieceTokenizerOperation()
339 SPieceTokenizerOutType out_type) in SentencePieceTokenizerOperation()
343 if (out_type_ != SPieceTokenizerOutType::kString && out_type_ != SPieceTokenizerOutType::kInt) { in ValidateParams()
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/include/dataset/
Dtext.h410 mindspore::dataset::SPieceTokenizerOutType out_type);
415 …SentencePieceTokenizer(const std::string &vocab_path, mindspore::dataset::SPieceTokenizerOutType o… in SentencePieceTokenizer()
421 …okenizer(const std::vector<char> &vocab_path, mindspore::dataset::SPieceTokenizerOutType out_type);
Dconstants.h139 enum class SPieceTokenizerOutType { enum