Home
last modified time | relevance | path

Searched refs:VOCAB_FILE (Results 1 – 3 of 3) sorted by relevance

/third_party/mindspore/tests/ut/python/dataset/
Dtest_sentencepiece_tokenizer.py21 VOCAB_FILE = "../data/dataset/test_sentencepiece/botchan.txt" variable
26 …vocab = text.SentencePieceVocab.from_file([VOCAB_FILE], 5000, 0.9995, SentencePieceModel.UNIGRAM, …
33 …vocab = text.SentencePieceVocab.from_file([VOCAB_FILE], 5000, 0.9995, SentencePieceModel.UNIGRAM, …
45 … vocab = text.SentencePieceVocab.from_file([VOCAB_FILE], 5000, 0.9995, SentencePieceModel.BPE, {})
57 … vocab = text.SentencePieceVocab.from_file([VOCAB_FILE], 5000, 0.9995, SentencePieceModel.CHAR, {})
70 … vocab = text.SentencePieceVocab.from_file([VOCAB_FILE], 5000, 0.9995, SentencePieceModel.WORD, {})
82 …vocab = text.SentencePieceVocab.from_file([VOCAB_FILE], 5000, 0.9995, SentencePieceModel.UNIGRAM, …
94 …vocab = text.SentencePieceVocab.from_file([VOCAB_FILE], 5000, 0.9995, SentencePieceModel.UNIGRAM, …
107 …vocab = text.SentencePieceVocab.from_file([VOCAB_FILE], 5000, 0.9995, SentencePieceModel.UNIGRAM, …
120 data = ds.TextFileDataset(VOCAB_FILE, shuffle=False)
[all …]
Dtest_nlp.py23 VOCAB_FILE = "../data/dataset/testVocab/vocab_list.txt" variable
31 with open(VOCAB_FILE, 'r') as f:
36 vocab = text.Vocab.from_file(VOCAB_FILE, ",", special_tokens=["<pad>", "<unk>"])
48 with open(VOCAB_FILE, 'r') as f:
54 vocab = text.Vocab.from_file(VOCAB_FILE, ",")
Dtest_vocab.py25 VOCAB_FILE = "../data/dataset/testVocab/vocab_list.txt" variable
52 vocab = text.Vocab.from_file(VOCAB_FILE, ",", None, ["<pad>", "<unk>"], True)