| /third_party/mindspore/tests/ut/python/dataset/ |
| D | test_text_jieba_tokenizer.py | 19 from mindspore.dataset.text import JiebaMode, to_str
34 jieba_op1 = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
35 jieba_op2 = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.HMM)
54 jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
68 jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.HMM)
81 jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MIX)
95 jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
110 jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
123 jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
134 jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
[all …]
|
| D | test_nlp.py | 30 jieba_op = text.JiebaTokenizer(HMM_FILE, MP_FILE, mode=text.JiebaMode.MP)
47 jieba_op = text.JiebaTokenizer(HMM_FILE, MP_FILE, mode=text.JiebaMode.MP)
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/kernels/ |
| D | jieba_tokenizer_op.h | 42 …p(const std::string &hmm_path, const std::string &mp_path, const JiebaMode &mode = JiebaMode::kMix,
61 JiebaMode jieba_mode_;
|
| D | jieba_tokenizer_op.cc | 26 …:JiebaTokenizerOp(const std::string &hmm_path, const std::string &dict_path, const JiebaMode &mode, in JiebaTokenizerOp()
40 if (jieba_mode_ == JiebaMode::kMp) { in Tokenize()
43 } else if (jieba_mode_ == JiebaMode::kHmm) { in Tokenize()
|
| /third_party/mindspore/mindspore/dataset/text/ |
| D | transforms.py | 50 from .utils import JiebaMode, NormalizeForm, to_str, SPieceTokenizerOutType, SPieceTokenizerLoadType
71 JiebaMode.MIX: cde.JiebaMode.DE_JIEBA_MIX,
72 JiebaMode.MP: cde.JiebaMode.DE_JIEBA_MP,
73 JiebaMode.HMM: cde.JiebaMode.DE_JIEBA_HMM
123 def __init__(self, hmm_path, mp_path, mode=JiebaMode.MIX, with_offsets=False):
124 if not isinstance(mode, JiebaMode):
|
| D | __init__.py | 30 from .utils import to_str, to_bytes, JiebaMode, Vocab, NormalizeForm, SentencePieceVocab, SentenceP…
|
| D | utils.py | 299 class JiebaMode(IntEnum): class
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/ir/ |
| D | bindings.cc | 141 …py::init([](const std::string &hmm_path, const std::string &mp_path, const JiebaMode &mode, bool w… in __anon5546ea161202()
246 PYBIND_REGISTER(JiebaMode, 0, ([](const py::module *m) { in __anon5546ea162602()
247 (void)py::enum_<JiebaMode>(*m, "JiebaMode", py::arithmetic()) in __anon5546ea162602()
248 .value("DE_JIEBA_MIX", JiebaMode::kMix) in __anon5546ea162602()
249 .value("DE_JIEBA_MP", JiebaMode::kMp) in __anon5546ea162602()
250 .value("DE_JIEBA_HMM", JiebaMode::kHmm) in __anon5546ea162602()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/ir/kernels/ |
| D | text_ir.cc | 146 const JiebaMode &mode, bool with_offsets) in JiebaTokenizerOperation()
162 if (mode_ != JiebaMode::kMix && mode_ != JiebaMode::kMp && mode_ != JiebaMode::kHmm) { in ValidateParams()
|
| D | text_ir.h | 121 …aTokenizerOperation(const std::string &hmm_path, const std::string &mp_path, const JiebaMode &mode,
137 JiebaMode mode_;
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/include/dataset/ |
| D | text.h | 165 …r(const std::string &hmm_path, const std::string &mp_path, const JiebaMode &mode = JiebaMode::kMix,
180 …kenizer(const std::vector<char> &hmm_path, const std::vector<char> &mp_path, const JiebaMode &mode,
|
| D | constants.h | 132 enum class JiebaMode { enum
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/api/ |
| D | text.cc | 114 …Data(const std::vector<char> &hmm_path, const std::vector<char> &mp_path, const JiebaMode &mode, b… in Data()
122 JiebaMode mode_;
128 const JiebaMode &mode, bool with_offsets) in JiebaTokenizer()
|
| /third_party/mindspore/tests/ut/cpp/dataset/ |
| D | c_api_text_test.cc | 777 std::make_shared<text::JiebaTokenizer>(hmm_path, mp_path, JiebaMode::kMp); in TEST_F()
826 std::make_shared<text::JiebaTokenizer>(hmm_path, mp_path, JiebaMode::kHmm); in TEST_F()
875 std::make_shared<text::JiebaTokenizer>(hmm_path, mp_path, JiebaMode::kMp, true); in TEST_F()
945 std::make_shared<text::JiebaTokenizer>("", mp_path, JiebaMode::kMp); in TEST_F()
970 std::make_shared<text::JiebaTokenizer>(hmm_path, "", JiebaMode::kMp); in TEST_F()
996 std::make_shared<text::JiebaTokenizer>(hmm_path_invalid, mp_path, JiebaMode::kMp); in TEST_F()
1022 std::make_shared<text::JiebaTokenizer>(hmm_path, mp_path_invalid, JiebaMode::kMp); in TEST_F()
1047 std::make_shared<text::JiebaTokenizer>(hmm_path, mp_path, JiebaMode::kMp); in TEST_F()
1099 std::make_shared<text::JiebaTokenizer>(hmm_path, mp_path, JiebaMode::kMp); in TEST_F()
1151 std::make_shared<text::JiebaTokenizer>(hmm_path, mp_path, JiebaMode::kMp); in TEST_F()
[all …]
|