Searched refs:vocab_file (Results 1 – 7 of 7) sorted by relevance
| /third_party/mindspore/tests/st/fl/albert/src/ |
| D | tokenization.py | 30 def load_vocab(vocab_file, vocab_map_ids_path=None): argument
43 with open(vocab_file, "r", encoding="utf-8") as reader:
53 with open(vocab_file, "r", encoding="utf-8") as reader:
76 …def __init__(self, vocab_file, do_lower_case=True, max_len=None, do_basic_tokenize=True, basic_onl… argument
92 self.vocab = load_vocab(vocab_file)
134 vocab_file = os.path.join(vocab_path, 'vocab.txt')
137 with open(vocab_file, "w", encoding="utf-8") as writer:
143 return vocab_file
449 def __init__(self, vocab_file, do_lower_case=True, max_len=None, argument
464 super().__init__(vocab_file, do_lower_case, max_len, never_split)
[all …]
|
| /third_party/mindspore/tests/st/networks/models/bert/src/ |
| D | cluener_evaluation.py | 27 vocab_file = "./vocab.txt" variable
28 tokenizer_ = tokenization.FullTokenizer(vocab_file=vocab_file)
|
| /third_party/mindspore/tests/ut/cpp/dataset/ |
| D | c_api_text_sentence_piece_vocab_test.cc | 42 std::string vocab_file = datasets_root_path_ + "/test_sentencepiece/botchan.txt"; in TEST_F() local
43 std::shared_ptr<Dataset> ds_vocab = TextFile({vocab_file}, 0, ShuffleMode::kFalse); in TEST_F()
101 std::string vocab_file = datasets_root_path_ + "/test_sentencepiece/botchan.txt"; in TEST_F() local
102 std::shared_ptr<Dataset> ds_vocab = TextFile({vocab_file}, 0, ShuffleMode::kFalse); in TEST_F()
164 std::string vocab_file = datasets_root_path_ + "/test_sentencepiece/botchan.txt"; in TEST_F() local
165 std::shared_ptr<Dataset> ds_vocab = TextFile({vocab_file}, 0, ShuffleMode::kFalse); in TEST_F()
|
| /third_party/mindspore/tests/ut/python/mindrecord/ |
| D | utils.py | 174 def get_nlp_data(dir_name, vocab_file, num): argument
197 dictionary = load_vocab(vocab_file)
228 def load_vocab(vocab_file): argument
233 with open(vocab_file) as reader:
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/ir/kernels/ |
| D | text_ir.cc | 357 Path vocab_file(real_vocab_path); in ValidateParams() local
358 if (!vocab_file.Exists() || vocab_file.IsDirectory()) { in ValidateParams()
363 if (access(vocab_file.ToString().c_str(), R_OK) == -1) { in ValidateParams()
377 Path vocab_file(vocab_path_); in Build() local
378 std::string model_path = vocab_file.ParentPath(); in Build()
379 std::string model_filename = vocab_file.Basename(); in Build()
|
| /third_party/mindspore/tests/ut/python/dataset/ |
| D | test_minddataset_padded.py | 617 def get_nlp_data(dir_name, vocab_file, num): argument
640 dictionary = load_vocab(vocab_file)
671 def load_vocab(vocab_file): argument
676 with open(vocab_file) as reader:
|
| D | test_minddataset.py | 1034 def get_nlp_data(dir_name, vocab_file, num): argument
1057 dictionary = load_vocab(vocab_file)
1088 def load_vocab(vocab_file): argument
1093 with open(vocab_file) as reader:
|