Home
last modified time | relevance | path

Searched refs:vocab_size (Results 1 – 25 of 54) sorted by relevance

123

/third_party/mindspore/mindspore/dataset/text/
Dutils.py104 …def from_file(cls, file_path, delimiter="", vocab_size=None, special_tokens=None, special_first=Tr… argument
125 if vocab_size is None:
126 vocab_size = -1
129 return super().from_file(file_path, delimiter, vocab_size, special_tokens, special_first)
158 def from_dataset(cls, dataset, col_names, vocab_size, character_coverage, model_type, params): argument
192 return dataset.build_sentencepiece_vocab(col_names, vocab_size, character_coverage,
197 def from_file(cls, file_path, vocab_size, character_coverage, model_type, params): argument
234 return super().from_file(file_path, vocab_size, character_coverage,
Dvalidators.py65 …[file_path, delimiter, vocab_size, special_tokens, special_first], _ = parse_user_args(method, *ar…
70 if vocab_size is not None:
71 check_positive(vocab_size, "vocab_size")
446 …[_, col_names, vocab_size, character_coverage, model_type, params], _ = parse_user_args(method, *a…
451 if vocab_size is not None:
452 check_uint32(vocab_size, "vocab_size")
476 …[file_path, vocab_size, character_coverage, model_type, params], _ = parse_user_args(method, *args…
481 if vocab_size is not None:
482 check_uint32(vocab_size, "vocab_size")
/third_party/mindspore/tests/st/dynamic_shape/
Dtest_dynamic_shape_embedding.py27 def __init__(self, vocab_size, embedding_size, target="CPU"): argument
30 nn.EmbeddingLookup(vocab_size=vocab_size,
48 net1 = NetWithEmbeddingLookUp(vocab_size=8, embedding_size=8, target="CPU")
56 net2 = NetWithEmbeddingLookUp(vocab_size=8, embedding_size=8, target="CPU")
/third_party/mindspore/tests/st/auto_parallel/
Dmultifieldembeddinglookup_parallel.py178 def __init__(self, vocab_size, embedding_size, field_size, argument
181 self.embedding = embedding(vocab_size=vocab_size,
202 def __init__(self, vocab_size, embedding_size, field_size, argument
204 self.vocab_size = vocab_size
267 … parallel_mode_net = MultiHotNet(vocab_size=self.vocab_size, embedding_size=self.embedding_size,
275 … stand_alone_net = MultiHotNet(vocab_size=self.vocab_size, embedding_size=self.embedding_size,
283 standalone_net = MultiHotNet(vocab_size=self.vocab_size, embedding_size=self.embedding_size,
287 parallel_net = MultiHotNet(vocab_size=self.vocab_size, embedding_size=self.embedding_size,
302 …fact = ParallelMultiHotFactory(vocab_size=32, embedding_size=64, field_size=64, param_init='one', …
/third_party/mindspore/tests/mindspore_test_framework/apps/
Dtest_bert_parts.py133 'block': get_output_cell(EmbeddingLookup(vocab_size=32000,
141 'block': get_output_cell(EmbeddingLookup(vocab_size=32000,
149 'block': get_output_cell(EmbeddingLookup(vocab_size=32000,
157 'block': get_output_cell(EmbeddingLookup(vocab_size=32000,
165 'block': EmbeddingLookup(vocab_size=32000,
173 'block': (get_output_cell(EmbeddingLookup(vocab_size=32000,
182 'block': (EmbeddingLookup(vocab_size=32000,
191 'block': (EmbeddingLookup(vocab_size=32000,
Dtest_bert_check_gradient.py286 'block': EmbeddingLookup(vocab_size=21128,
314 vocab_size=21128,
384 vocab_size=21128,
Dtest_bert_compare_with_npy.py467 'block': EmbeddingLookup(vocab_size=21128,
571 vocab_size=21128,
/third_party/mindspore/mindspore/nn/layer/
Dembedding.py98 def __init__(self, vocab_size, embedding_size, use_one_hot=False, embedding_table='normal', argument
102 self.vocab_size = validator.check_value_type('vocab_size', vocab_size, [int], self.cls_name)
108 self.init_tensor = initializer(embedding_table, [vocab_size, embedding_size])
111 self.padding_idx = validator.check_int_range(padding_idx, 0, vocab_size, Rel.INC_BOTH,
136 one_hot_ids = self.one_hot(flat_ids, self.vocab_size, self.on_value, self.off_value)
146 …self.vocab_size, self.embedding_size, self.use_one_hot, self.embedding_table, self.dtype, self.pad…
223 def __init__(self, vocab_size, embedding_size, param_init='normal', argument
229 self.vocab_size = validator.check_positive_int(vocab_size, 'vocab_size')
248 … self.embedding_table = Parameter(initializer(param_init, [self.vocab_size, self.embedding_size]),
260 self._set_voacb_cache_enable_for_ps(vocab_cache_size, embedding_size, vocab_size)
[all …]
Dthor_layer.py570 def __init__(self, vocab_size, embedding_size, use_one_hot=False, embedding_table='normal', argument
574 self.vocab_size = Validator.check_value_type('vocab_size', vocab_size, [int], self.cls_name)
580 self.init_tensor = initializer(embedding_table, [vocab_size, embedding_size])
583 self.padding_idx = Validator.check_int_range(padding_idx, 0, vocab_size, Rel.INC_BOTH,
599 self.matrix_a = Parameter(Tensor(np.zeros([vocab_size]).astype(np.float32)),
631 one_hot_ids = self.one_hot(flat_ids, self.vocab_size, self.on_value, self.off_value)
635 one_hot_ids = self.one_hot(flat_ids, self.vocab_size, self.on_value, self.off_value)
648 …self.vocab_size, self.embedding_size, self.use_one_hot, self.embedding_table, self.dtype, self.pad…
720 def __init__(self, vocab_size, embedding_size, param_init='normal', argument
725 self.vocab_size = Validator.check_positive_int(vocab_size, 'vocab_size', self.cls_name)
[all …]
/third_party/mindspore/tests/st/ps/part_ps/
Dtest_ps_embedding_heterogeneous_conv2d_adam.py51 def __init__(self, in_channels, out_channels, kernel_size, vocab_size, embedding_size, argument
59 self.embedding_lookup = EmbeddingLookup(vocab_size=vocab_size,
126 kernel_size=5, vocab_size=5, embedding_size=1, output_channels=3072, argument
131 self.vocab_size = vocab_size
141 net = Menet(self.in_channels, self.out_channels, self.kernel_size, self.vocab_size,
157 net = Menet(self.in_channels, self.out_channels, self.kernel_size, self.vocab_size,
/third_party/mindspore/tests/st/networks/
Dtest_gpu_lstm.py62 def __init__(self, vocab_size, embed_size, num_hiddens, num_layers, argument
70 …self.embedding = nn.Embedding(vocab_size, embed_size, use_one_hot=False, embedding_table=Tensor(we…
118 vocab_size = 252193
121 weight = np.ones((vocab_size + 1, embed_size)).astype(np.float32)
123 net = SentimentNet(vocab_size=(vocab_size + 1), embed_size=embed_size,
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/
Dvocab.cc135 … Vocab::BuildFromFileCpp(const std::string &path, const std::string &delimiter, int32_t vocab_size, in BuildFromFileCpp() argument
146 !(vocab_size < 0 && vocab_size != -1), in BuildFromFileCpp()
147 …file: vocab_size should be either -1 or positive integer, but got: " + std::to_string(vocab_size)); in BuildFromFileCpp()
184 if (word2id.size() == vocab_size) break; in BuildFromFileCpp()
197 …tus Vocab::BuildFromFile(const std::string &path, const std::string &delimiter, int32_t vocab_size, in BuildFromFile() argument
229 if (word2id.size() == vocab_size) break; in BuildFromFile()
Dvocab.h62 …tic Status BuildFromFile(const std::string &path, const std::string &delimiter, int32_t vocab_size,
90 … Status BuildFromFileCpp(const std::string &path, const std::string &delimiter, int32_t vocab_size,
Dsentence_piece_vocab.cc33 …tencePieceVocab::BuildFromFile(const std::vector<std::string> &path_list, const int32_t vocab_size, in BuildFromFile() argument
50 unorder_map["vocab_size"] = std::to_string(vocab_size); in BuildFromFile()
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/
Dbindings.cc39 [](const std::string &path, const std::string &dlm, int32_t vocab_size, in __anonf58a86e70102()
42 … THROW_IF_ERROR(Vocab::BuildFromFile(path, dlm, vocab_size, special_tokens, special_first, &v)); in __anonf58a86e70102()
56 … [](const py::list &paths, const int32_t vocab_size, const float character_coverage, in __anonf58a86e70502() argument
73 … path_list, vocab_size, character_coverage, model_type, param_map, &v)); in __anonf58a86e70502()
/third_party/mindspore/tests/st/networks/models/bert/src/
Dconfig.py59 vocab_size=21128,
80 vocab_size=21128,
101 vocab_size=30522,
Dbert_model.py65 vocab_size=32000, argument
84 self.vocab_size = vocab_size
116 vocab_size, argument
122 self.vocab_size = vocab_size
126 [vocab_size, embedding_size]),
142 one_hot_ids = self.one_hot(flat_ids, self.vocab_size, self.on_value, self.off_value)
323 self.vocab_size = max_relative_position * 2 + 1
328 [self.vocab_size, self.depth]),
334 self.one_hot = nn.OneHot(depth=self.vocab_size)
870 vocab_size=config.vocab_size,
/third_party/mindspore/mindspore/train/train_thor/
Dconvert_utils.py74 new_subcell = nn.EmbeddingThor(vocab_size=subcell.vocab_size,
86 new_subcell = nn.EmbeddingLookupThor(vocab_size=subcell.vocab_size,
/third_party/mindspore/tests/st/model_zoo_tests/wide_and_deep/python_file_for_ci/
Dconfig.py55 self.vocab_size = 184968
85 self.vocab_size = args.vocab_size
/third_party/mindspore/tests/st/fl/albert/src/
Dconfig.py64 vocab_size=11682,
100 vocab_size=11682,
Ddataset.py151 …def __init__(self, batch_size, max_seq_length, vocab_size, keep_first_unchange=True, keep_last_unc… argument
154 self.vocab_size = vocab_size
172 self.replace_tensor[i, j] = np.random.randint(0, self.vocab_size)
Dmodel.py60 vocab_size=21128, argument
93 self.vocab_size = vocab_size
137 self.vocab_size = config.vocab_size
141 [config.vocab_size, config.embedding_size]),
157 one_hot_ids = self.one_hot(flat_ids, self.vocab_size, self.on_value, self.off_value)
318 self.vocab_size = max_relative_position * 2 + 1
322 [self.vocab_size, self.depth]),
341 flat_relative_positions_matrix, self.vocab_size, self.on_value, self.off_value)
796 config.vocab_size,
Dcell_wrapper.py177 def __init__(self, network, vocab_size=21128): argument
180 self.vocab_size = vocab_size
186 prediction_scores = self.reshape(prediction_scores, (-1, self.vocab_size))
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/
Dbuild_sentence_piece_vocab_node.cc32 … const std::vector<std::string> &col_names, int32_t vocab_size, in BuildSentenceVocabNode() argument
37 vocab_size_(vocab_size), in BuildSentenceVocabNode()
/third_party/mindspore/tests/ut/python/dataset/
Dtest_vocab.py152 def test_config(lookup_str, vocab_size, special_tokens, special_first): argument
154 …vocab = text.Vocab.from_file(SIMPLE_VOCAB_FILE, vocab_size=vocab_size, special_tokens=special_toke…

123