1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include <memory>
17 #include <vector>
18 #include <string>
19
20 #include "common/common.h"
21 #include "include/api/status.h"
22 #include "minddata/dataset/include/dataset/datasets.h"
23 #include "minddata/dataset/include/dataset/text.h"
24 #include "minddata/dataset/include/dataset/transforms.h"
25 #include "minddata/dataset/text/vocab.h"
26
27 using namespace mindspore::dataset;
28 using mindspore::Status;
29 using mindspore::dataset::DataType;
30 using mindspore::dataset::ShuffleMode;
31 using mindspore::dataset::Tensor;
32 using mindspore::dataset::Vocab;
33
34 class MindDataTestPipeline : public UT::DatasetOpTesting {
35 protected:
36 };
37
38 // Macro to compare 2 MSTensors as not equal; compare datasize only
39 #define EXPECT_MSTENSOR_DATA_NE(_mstensor1, _mstensor2) \
40 do { \
41 EXPECT_NE(_mstensor1.DataSize(), _mstensor2.DataSize()); \
42 } while (false)
43
TEST_F(MindDataTestPipeline,TestVocabLookupOp)44 TEST_F(MindDataTestPipeline, TestVocabLookupOp) {
45 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabLookupOp.";
46
47 // Create a TextFile dataset
48 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
49 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
50 EXPECT_NE(ds, nullptr);
51
52 // Create a vocab from vector
53 std::vector<std::string> list = {"home", "IS", "behind", "the", "world", "ahead", "!"};
54 std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
55 Status s = Vocab::BuildFromVector(list, {"<pad>", "<unk>"}, true, &vocab);
56 EXPECT_EQ(s, Status::OK());
57
58 // Create Lookup operation on ds
59 std::shared_ptr<TensorTransform> lookup =
60 std::make_shared<text::Lookup>(vocab, "<unk>", mindspore::DataType::kNumberTypeInt32);
61 EXPECT_NE(lookup, nullptr);
62
63 // Create Map operation on ds
64 ds = ds->Map({lookup}, {"text"});
65 EXPECT_NE(ds, nullptr);
66
67 // Create an iterator over the result of the above dataset
68 // This will trigger the creation of the Execution Tree and launch it.
69 std::shared_ptr<Iterator> iter = ds->CreateIterator();
70 EXPECT_NE(iter, nullptr);
71
72 // Iterate the dataset and get each row
73 std::unordered_map<std::string, mindspore::MSTensor> row;
74 ASSERT_OK(iter->GetNextRow(&row));
75
76 uint64_t i = 0;
77 std::vector<int32_t> expected = {2, 1, 4, 5, 6, 7};
78 while (row.size() != 0) {
79 auto ind = row["text"];
80 MS_LOG(INFO) << ind.Shape();
81 TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
82 std::shared_ptr<Tensor> de_expected_item;
83 ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_item));
84 mindspore::MSTensor ms_expected_item =
85 mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_item));
86 EXPECT_MSTENSOR_EQ(ind, ms_expected_item);
87
88 ASSERT_OK(iter->GetNextRow(&row));
89 i++;
90 }
91
92 EXPECT_EQ(i, 6);
93
94 // Manually terminate the pipeline
95 iter->Stop();
96 }
97
TEST_F(MindDataTestPipeline,TestVocabLookupOpEmptyString)98 TEST_F(MindDataTestPipeline, TestVocabLookupOpEmptyString) {
99 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabLookupOpEmptyString.";
100
101 // Create a TextFile dataset
102 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
103 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
104 EXPECT_NE(ds, nullptr);
105
106 // Create a vocab from vector
107 std::vector<std::string> list = {"home", "IS", "behind", "the", "world", "ahead", "!"};
108 std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
109 Status s = Vocab::BuildFromVector(list, {"<pad>", ""}, true, &vocab);
110 EXPECT_EQ(s, Status::OK());
111
112 // Create Lookup operation on ds
113 std::shared_ptr<TensorTransform> lookup =
114 std::make_shared<text::Lookup>(vocab, "", mindspore::DataType::kNumberTypeInt32);
115 EXPECT_NE(lookup, nullptr);
116
117 // Create Map operation on ds
118 ds = ds->Map({lookup}, {"text"});
119 EXPECT_NE(ds, nullptr);
120
121 // Create an iterator over the result of the above dataset
122 // This will trigger the creation of the Execution Tree and launch it.
123 std::shared_ptr<Iterator> iter = ds->CreateIterator();
124 EXPECT_NE(iter, nullptr);
125
126 // Iterate the dataset and get each row
127 std::unordered_map<std::string, mindspore::MSTensor> row;
128 ASSERT_OK(iter->GetNextRow(&row));
129
130 uint64_t i = 0;
131 std::vector<int32_t> expected = {2, 1, 4, 5, 6, 7};
132 while (row.size() != 0) {
133 auto ind = row["text"];
134 MS_LOG(INFO) << ind.Shape();
135 TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
136 std::shared_ptr<Tensor> de_expected_item;
137 ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_item));
138 mindspore::MSTensor ms_expected_item =
139 mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_item));
140 EXPECT_MSTENSOR_EQ(ind, ms_expected_item);
141
142 ASSERT_OK(iter->GetNextRow(&row));
143 i++;
144 }
145
146 EXPECT_EQ(i, 6);
147
148 // Manually terminate the pipeline
149 iter->Stop();
150 }
151
TEST_F(MindDataTestPipeline,TestVocabLookupBool)152 TEST_F(MindDataTestPipeline, TestVocabLookupBool) {
153 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabLookupBool.";
154 // Invoke Lookup with Bool data_type
155
156 // Create a TextFile dataset
157 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
158 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
159 EXPECT_NE(ds, nullptr);
160
161 // Create a vocab from vector
162 std::vector<std::string> list = {"home", "IS", "behind", "the", "world", "ahead", "!"};
163 std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
164 Status s = Vocab::BuildFromVector(list, {"<pad>", "<unk>"}, true, &vocab);
165 EXPECT_EQ(s, Status::OK());
166
167 // Create Lookup operation on ds
168 std::shared_ptr<TensorTransform> lookup =
169 std::make_shared<text::Lookup>(vocab, "<unk>", mindspore::DataType::kNumberTypeBool);
170 EXPECT_NE(lookup, nullptr);
171
172 // Create Map operation on ds
173 ds = ds->Map({lookup}, {"text"});
174 EXPECT_NE(ds, nullptr);
175
176 // Create an iterator over the result of the above dataset
177 // This will trigger the creation of the Execution Tree and launch it.
178 std::shared_ptr<Iterator> iter = ds->CreateIterator();
179 EXPECT_NE(iter, nullptr);
180
181 // Iterate the dataset and get each row
182 std::unordered_map<std::string, mindspore::MSTensor> row;
183 ASSERT_OK(iter->GetNextRow(&row));
184
185 uint64_t i = 0;
186 while (row.size() != 0) {
187 auto ind = row["text"];
188 MS_LOG(INFO) << ind.Shape();
189 TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
190
191 ASSERT_OK(iter->GetNextRow(&row));
192 i++;
193 }
194
195 EXPECT_EQ(i, 6);
196
197 // Manually terminate the pipeline
198 iter->Stop();
199 }
200
TEST_F(MindDataTestPipeline,TestVocabLookupOpFail1)201 TEST_F(MindDataTestPipeline, TestVocabLookupOpFail1) {
202 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabLookupOpFail1.";
203 // Create a TextFile Dataset
204 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
205 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
206 EXPECT_NE(ds, nullptr);
207
208 // Build vocab from vector
209 std::vector<std::string> list = {"home", "IS", "behind", "the", "world", "ahead", "!"};
210 std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
211 Status s = Vocab::BuildFromVector(list, {}, true, &vocab);
212 EXPECT_EQ(s, Status::OK());
213
214 // Create lookup op for ds
215 std::shared_ptr<TensorTransform> lookup =
216 std::make_shared<text::Lookup>(vocab, "<unk>", mindspore::DataType::kNumberTypeInt32);
217 EXPECT_NE(lookup, nullptr);
218
219 // Create a Map operation on ds
220 ds = ds->Map({lookup});
221 EXPECT_NE(ds, nullptr);
222
223 std::shared_ptr<Iterator> iter = ds->CreateIterator();
224 // Expect failure: invalid Lookup input ("<unk>" is not a word of vocab)
225 EXPECT_EQ(iter, nullptr);
226 }
227
TEST_F(MindDataTestPipeline,TestVocabLookupOpFail2)228 TEST_F(MindDataTestPipeline, TestVocabLookupOpFail2) {
229 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabLookupOpFail2.";
230 // Create a TextFile Dataset
231 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
232 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
233 EXPECT_NE(ds, nullptr);
234
235 // Vocab has nothing
236 std::shared_ptr<Vocab> vocab;
237
238 // Create lookup op
239 std::shared_ptr<TensorTransform> lookup =
240 std::make_shared<text::Lookup>(vocab, "", mindspore::DataType::kNumberTypeInt32);
241 EXPECT_NE(lookup, nullptr);
242
243 // Create a Map operation on ds
244 ds = ds->Map({lookup});
245 EXPECT_NE(ds, nullptr);
246
247 std::shared_ptr<Iterator> iter = ds->CreateIterator();
248 // Expect failure: invalid Lookup input (vocab is null)
249 EXPECT_EQ(iter, nullptr);
250 }
251
TEST_F(MindDataTestPipeline,TestVocabLookupOpFail3DataType)252 TEST_F(MindDataTestPipeline, TestVocabLookupOpFail3DataType) {
253 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabLookupOpFail3DataType.";
254 // Create a TextFile Dataset
255 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
256 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
257 EXPECT_NE(ds, nullptr);
258
259 // Build vocab from vector
260 std::vector<std::string> list = {"home", "IS", "behind", "the", "world", "ahead", "!"};
261 std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
262 Status s = Vocab::BuildFromVector(list, {"<pad>", "<unk>"}, true, &vocab);
263 EXPECT_EQ(s, Status::OK());
264
265 // Create lookup op for ds
266 std::shared_ptr<TensorTransform> lookup =
267 std::make_shared<text::Lookup>(vocab, "", mindspore::DataType::kObjectTypeString);
268 EXPECT_NE(lookup, nullptr);
269
270 // Create a Map operation on ds
271 ds = ds->Map({lookup});
272 EXPECT_NE(ds, nullptr);
273
274 std::shared_ptr<Iterator> iter = ds->CreateIterator();
275 // Expect failure: invalid Lookup input (String is not valid for data_type)
276 EXPECT_EQ(iter, nullptr);
277 }
278
TEST_F(MindDataTestPipeline,TestVocabFromDataset)279 TEST_F(MindDataTestPipeline, TestVocabFromDataset) {
280 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabFromDataset.";
281
282 // Create a TextFile dataset
283 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
284 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
285 EXPECT_NE(ds, nullptr);
286
287 // Create vocab from dataset
288 std::shared_ptr<Vocab> vocab = ds->BuildVocab({"text"}, {0, std::numeric_limits<int64_t>::max()},
289 std::numeric_limits<int64_t>::max(), {"<pad>", "<unk>"}, true);
290 EXPECT_NE(vocab, nullptr);
291
292 // Check if vocab has words or not
293 int32_t home_index = vocab->Lookup("home");
294 EXPECT_EQ(home_index, 4);
295
296 // Create Lookup operation on ds
297 std::shared_ptr<TensorTransform> lookup =
298 std::make_shared<text::Lookup>(vocab, "<unk>", mindspore::DataType::kNumberTypeInt32);
299 EXPECT_NE(lookup, nullptr);
300
301 // Create Map operation on ds
302 ds = ds->Map({lookup}, {"text"});
303 EXPECT_NE(ds, nullptr);
304
305 // Create an iterator over the result of the above dataset
306 // This will trigger the creation of the Execution Tree and launch it.
307 std::shared_ptr<Iterator> iter = ds->CreateIterator();
308 EXPECT_NE(iter, nullptr);
309
310 // Iterate the dataset and get each row
311 std::unordered_map<std::string, mindspore::MSTensor> row;
312 ASSERT_OK(iter->GetNextRow(&row));
313
314 uint64_t i = 0;
315 std::vector<int32_t> expected = {4, 5, 3, 6, 7, 2};
316 while (row.size() != 0) {
317 auto ind = row["text"];
318 MS_LOG(INFO) << ind.Shape();
319 TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
320 std::shared_ptr<Tensor> de_expected_item;
321 ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_item));
322 mindspore::MSTensor ms_expected_item =
323 mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_item));
324 EXPECT_MSTENSOR_EQ(ind, ms_expected_item);
325
326 ASSERT_OK(iter->GetNextRow(&row));
327 i++;
328 }
329
330 EXPECT_EQ(i, 6);
331
332 // Manually terminate the pipeline
333 iter->Stop();
334 }
335
TEST_F(MindDataTestPipeline,TestVocabFromDatasetDefault)336 TEST_F(MindDataTestPipeline, TestVocabFromDatasetDefault) {
337 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabFromDatasetDefault.";
338
339 // Create a TextFile dataset
340 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
341 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
342 EXPECT_NE(ds, nullptr);
343
344 // Create vocab from dataset
345 std::shared_ptr<Vocab> vocab = ds->BuildVocab();
346 EXPECT_NE(vocab, nullptr);
347
348 // Check if vocab has words or not
349 int32_t home_index = vocab->Lookup("home");
350 EXPECT_EQ(home_index, 2);
351
352 // Create Lookup operation on ds
353 // Use default data_type parameter
354 std::shared_ptr<TensorTransform> lookup = std::make_shared<text::Lookup>(vocab, "home");
355 EXPECT_NE(lookup, nullptr);
356
357 // Create Map operation on ds
358 ds = ds->Map({lookup});
359 EXPECT_NE(ds, nullptr);
360
361 // Create an iterator over the result of the above dataset
362 // This will trigger the creation of the Execution Tree and launch it.
363 std::shared_ptr<Iterator> iter = ds->CreateIterator();
364 EXPECT_NE(iter, nullptr);
365
366 // Iterate the dataset and get each row
367 std::unordered_map<std::string, mindspore::MSTensor> row;
368 ASSERT_OK(iter->GetNextRow(&row));
369
370 uint64_t i = 0;
371 std::vector<int32_t> expected = {2, 3, 1, 4, 5, 0};
372 std::vector<int64_t> not_expected = {2, 3, 1, 4, 5, 0};
373 while (row.size() != 0) {
374 auto ind = row["text"];
375 MS_LOG(INFO) << ind.Shape();
376 TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
377
378 std::shared_ptr<Tensor> de_expected_item;
379 ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_item));
380 mindspore::MSTensor ms_expected_item =
381 mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_item));
382 EXPECT_MSTENSOR_EQ(ind, ms_expected_item);
383
384 std::shared_ptr<Tensor> de_not_expected_item;
385 ASSERT_OK(Tensor::CreateScalar(not_expected[i], &de_not_expected_item));
386 mindspore::MSTensor ms_not_expected_item =
387 mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_not_expected_item));
388 EXPECT_MSTENSOR_DATA_NE(ind, ms_not_expected_item);
389
390 ASSERT_OK(iter->GetNextRow(&row));
391 i++;
392 }
393
394 EXPECT_EQ(i, 6);
395
396 // Manually terminate the pipeline
397 iter->Stop();
398 }
399
TEST_F(MindDataTestPipeline,TestVocabFromDatasetFail1)400 TEST_F(MindDataTestPipeline, TestVocabFromDatasetFail1) {
401 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabFromDatasetFail1.";
402
403 // Create a TextFile dataset
404 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
405 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
406 EXPECT_NE(ds, nullptr);
407
408 // Create vocab from dataset
409 // Expected failure: top_k can not be negative
410 std::shared_ptr<Vocab> vocab =
411 ds->BuildVocab({"text"}, {0, std::numeric_limits<int64_t>::max()}, -2, {"<pad>", "<unk>"}, true);
412 EXPECT_EQ(vocab, nullptr);
413 }
414
TEST_F(MindDataTestPipeline,TestVocabFromDatasetFail2)415 TEST_F(MindDataTestPipeline, TestVocabFromDatasetFail2) {
416 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabFromDatasetFail2.";
417
418 // Create a TextFile dataset
419 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
420 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
421 EXPECT_NE(ds, nullptr);
422
423 // Create vocab from dataset
424 // Expected failure: frequency_range [a,b] should be 0 <= a <= b
425 std::shared_ptr<Vocab> vocab =
426 ds->BuildVocab({"text"}, {4, 1}, std::numeric_limits<int64_t>::max(), {"<pad>", "<unk>"}, true);
427 EXPECT_EQ(vocab, nullptr);
428 }
429
TEST_F(MindDataTestPipeline,TestVocabFromDatasetFail3)430 TEST_F(MindDataTestPipeline, TestVocabFromDatasetFail3) {
431 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabFromDatasetFail3.";
432
433 // Create a TextFile dataset
434 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
435 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
436 EXPECT_NE(ds, nullptr);
437
438 // Create vocab from dataset
439 // Expected failure: column name does not exist in ds
440 std::shared_ptr<Vocab> vocab = ds->BuildVocab({"ColumnNotExist"});
441 EXPECT_EQ(vocab, nullptr);
442 }
443
TEST_F(MindDataTestPipeline,TestVocabFromDatasetFail4)444 TEST_F(MindDataTestPipeline, TestVocabFromDatasetFail4) {
445 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabFromDatasetFail4.";
446
447 // Create a TextFile dataset
448 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
449 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
450 EXPECT_NE(ds, nullptr);
451
452 // Create vocab from dataset
453 // Expected failure: special tokens are already in the dataset
454 std::shared_ptr<Vocab> vocab =
455 ds->BuildVocab({"text"}, {0, std::numeric_limits<int64_t>::max()}, std::numeric_limits<int64_t>::max(), {"world"});
456 EXPECT_EQ(vocab, nullptr);
457 }
458
TEST_F(MindDataTestPipeline,TestVocabFromDatasetInt64)459 TEST_F(MindDataTestPipeline, TestVocabFromDatasetInt64) {
460 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabFromDatasetInt64.";
461
462 // Create a TextFile dataset
463 std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
464 std::shared_ptr<Dataset> ds = TextFile({data_file}, 0, ShuffleMode::kFalse);
465 EXPECT_NE(ds, nullptr);
466
467 // Create vocab from dataset
468 std::shared_ptr<Vocab> vocab = ds->BuildVocab();
469 EXPECT_NE(vocab, nullptr);
470
471 // Check if vocab has words or not
472 int32_t home_index = vocab->Lookup("home");
473 EXPECT_EQ(home_index, 2);
474
475 // Create Lookup operation on ds
476 std::shared_ptr<TensorTransform> lookup =
477 std::make_shared<text::Lookup>(vocab, "home", mindspore::DataType::kNumberTypeInt64);
478 EXPECT_NE(lookup, nullptr);
479
480 // Create Map operation on ds
481 ds = ds->Map({lookup});
482 EXPECT_NE(ds, nullptr);
483
484 // Create an iterator over the result of the above dataset
485 // This will trigger the creation of the Execution Tree and launch it.
486 std::shared_ptr<Iterator> iter = ds->CreateIterator();
487 EXPECT_NE(iter, nullptr);
488
489 // Iterate the dataset and get each row
490 std::unordered_map<std::string, mindspore::MSTensor> row;
491 ASSERT_OK(iter->GetNextRow(&row));
492
493 uint64_t i = 0;
494 std::vector<int64_t> expected = {2, 3, 1, 4, 5, 0};
495 std::vector<int8_t> not_expected = {2, 3, 1, 4, 5, 0};
496 while (row.size() != 0) {
497 auto ind = row["text"];
498 MS_LOG(INFO) << ind.Shape();
499 TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
500
501 std::shared_ptr<Tensor> de_expected_item;
502 ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_item));
503 mindspore::MSTensor ms_expected_item =
504 mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_item));
505 EXPECT_MSTENSOR_EQ(ind, ms_expected_item);
506
507 std::shared_ptr<Tensor> de_not_expected_item;
508 ASSERT_OK(Tensor::CreateScalar(not_expected[i], &de_not_expected_item));
509 mindspore::MSTensor ms_not_expected_item =
510 mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_not_expected_item));
511 EXPECT_MSTENSOR_DATA_NE(ind, ms_not_expected_item);
512
513 ASSERT_OK(iter->GetNextRow(&row));
514 i++;
515 }
516
517 EXPECT_EQ(i, 6);
518
519 // Manually terminate the pipeline
520 iter->Stop();
521 }
522