1 /**
2 * Copyright 2020 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include <fstream>
17 #include <iostream>
18 #include <memory>
19 #include <string>
20 #include "common/common.h"
21 #include "minddata/dataset/core/client.h"
22 #include "minddata/dataset/core/global_context.h"
23 #include "minddata/dataset/core/tensor.h"
24 #include "minddata/dataset/core/tensor_shape.h"
25 #include "minddata/dataset/core/data_type.h"
26 #include "minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.h"
27 #include "minddata/dataset/engine/datasetops/source/sampler/pk_sampler.h"
28 #include "minddata/dataset/engine/datasetops/source/sampler/random_sampler.h"
29 #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"
30 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
31 #include "minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
32 #include "minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h"
33 #include "minddata/dataset/include/dataset/data_helper.h"
34 #include "minddata/dataset/util/path.h"
35 #include "minddata/dataset/util/status.h"
36 #include "gtest/gtest.h"
37 #include "utils/log_adapter.h"
38 #include "securec.h"
39
40 using namespace mindspore::dataset;
41 using mindspore::MsLogLevel::ERROR;
42 using mindspore::ExceptionType::NoExceptionType;
43 using mindspore::LogStream;
44
45 class MindDataTestDataHelper : public UT::DatasetOpTesting {
46 protected:
47 };
48
TEST_F(MindDataTestDataHelper,MindDataTestHelper)49 TEST_F(MindDataTestDataHelper, MindDataTestHelper) {
50 std::string file_path = datasets_root_path_ + "/testAlbum/images/1.json";
51 DataHelper dh;
52 std::vector<std::string> new_label = {"3", "4"};
53 Status rc = dh.UpdateArray(file_path, "label", new_label);
54 if (rc.IsError()) {
55 MS_LOG(ERROR) << "Return code error detected during label update: " << ".";
56 EXPECT_TRUE(false);
57 }
58 }
59
TEST_F(MindDataTestDataHelper,MindDataTestAlbumGen)60 TEST_F(MindDataTestDataHelper, MindDataTestAlbumGen) {
61 std::string file_path = datasets_root_path_ + "/testAlbum/original";
62 std::string out_path = datasets_root_path_ + "/testAlbum/testout";
63 DataHelper dh;
64 Status rc = dh.CreateAlbum(file_path, out_path);
65 if (rc.IsError()) {
66 MS_LOG(ERROR) << "Return code error detected during album generation: " << ".";
67 EXPECT_TRUE(false);
68 }
69 }
70
TEST_F(MindDataTestDataHelper,MindDataTestTemplateUpdateArrayInt)71 TEST_F(MindDataTestDataHelper, MindDataTestTemplateUpdateArrayInt) {
72 std::string file_path = datasets_root_path_ + "/testAlbum/testout/2.json";
73 DataHelper dh;
74 std::vector<int> new_label = {3, 4};
75 Status rc = dh.UpdateArray(file_path, "label", new_label);
76 if (rc.IsError()) {
77 MS_LOG(ERROR) << "Return code error detected during json int array update: " << ".";
78 EXPECT_TRUE(false);
79 }
80 }
81
TEST_F(MindDataTestDataHelper,MindDataTestTemplateUpdateArrayString)82 TEST_F(MindDataTestDataHelper, MindDataTestTemplateUpdateArrayString) {
83 std::string file_path = datasets_root_path_ + "/testAlbum/testout/3.json";
84 DataHelper dh;
85 std::vector<std::string> new_label = {"3", "4"};
86 Status rc = dh.UpdateArray(file_path, "label", new_label);
87 if (rc.IsError()) {
88 MS_LOG(ERROR) << "Return code error detected during json string array update: " << ".";
89 EXPECT_TRUE(false);
90 }
91 }
92
TEST_F(MindDataTestDataHelper,MindDataTestTemplateUpdateValueInt)93 TEST_F(MindDataTestDataHelper, MindDataTestTemplateUpdateValueInt) {
94 std::string file_path = datasets_root_path_ + "/testAlbum/testout/4.json";
95 DataHelper dh;
96 int new_label = 3;
97 Status rc = dh.UpdateValue(file_path, "label", new_label);
98 if (rc.IsError()) {
99 MS_LOG(ERROR) << "Return code error detected during json int update: " << ".";
100 EXPECT_TRUE(false);
101 }
102 }
103
TEST_F(MindDataTestDataHelper,MindDataTestTemplateUpdateString)104 TEST_F(MindDataTestDataHelper, MindDataTestTemplateUpdateString) {
105 std::string file_path = datasets_root_path_ + "/testAlbum/testout/5.json";
106 DataHelper dh;
107 std::string new_label = "new label";
108 Status rc = dh.UpdateValue(file_path, "label", new_label);
109 if (rc.IsError()) {
110 MS_LOG(ERROR) << "Return code error detected during json string update: " << ".";
111 EXPECT_TRUE(false);
112 }
113 }
114
TEST_F(MindDataTestDataHelper,MindDataTestDeleteKey)115 TEST_F(MindDataTestDataHelper, MindDataTestDeleteKey) {
116 std::string file_path = datasets_root_path_ + "/testAlbum/testout/5.json";
117 DataHelper dh;
118 Status rc = dh.RemoveKey(file_path, "label");
119 if (rc.IsError()) {
120 MS_LOG(ERROR) << "Return code error detected during json key remove: " << ".";
121 EXPECT_TRUE(false);
122 }
123 }
124
TEST_F(MindDataTestDataHelper,MindDataTestBinWrite)125 TEST_F(MindDataTestDataHelper, MindDataTestBinWrite) {
126 std::string file_path = datasets_root_path_ + "/testAlbum/1.bin";
127 DataHelper dh;
128 std::vector<float> bin_content = {3, 4};
129 Status rc = dh.WriteBinFile(file_path, bin_content);
130 if (rc.IsError()) {
131 MS_LOG(ERROR) << "Return code error detected during bin file write: " << ".";
132 EXPECT_TRUE(false);
133 }
134 }
135
TEST_F(MindDataTestDataHelper,MindDataTestBinWritePointer)136 TEST_F(MindDataTestDataHelper, MindDataTestBinWritePointer) {
137 std::string file_path = datasets_root_path_ + "/testAlbum/2.bin";
138 DataHelper dh;
139 std::vector<float> bin_content = {3, 4};
140
141 Status rc = dh.WriteBinFile(file_path, &bin_content[0], bin_content.size());
142 if (rc.IsError()) {
143 MS_LOG(ERROR) << "Return code error detected during binfile write: " << ".";
144 EXPECT_TRUE(false);
145 }
146 }
147
TEST_F(MindDataTestDataHelper,MindDataTestTensorWriteFloat)148 TEST_F(MindDataTestDataHelper, MindDataTestTensorWriteFloat) {
149 // create tensor
150 std::vector<float> y = {2.5, 3.0, 3.5, 4.0};
151 std::shared_ptr<Tensor> t;
152 Tensor::CreateFromVector(y, &t);
153 // create buffer using system mempool
154 DataHelper dh;
155 void *data = malloc(t->SizeInBytes());
156 if (data == nullptr) {
157 MS_LOG(ERROR) << "malloc failed";
158 ASSERT_TRUE(false);
159 }
160 auto bytes_copied = dh.DumpData(t->GetBuffer(), t->SizeInBytes(), data, t->SizeInBytes());
161 if (bytes_copied != t->SizeInBytes()) {
162 EXPECT_TRUE(false);
163 }
164 float *array = static_cast<float *>(data);
165 if (array[0] != 2.5) { EXPECT_TRUE(false); }
166 if (array[1] != 3.0) { EXPECT_TRUE(false); }
167 if (array[2] != 3.5) { EXPECT_TRUE(false); }
168 if (array[3] != 4.0) { EXPECT_TRUE(false); }
169 std::free(data);
170 }
171
TEST_F(MindDataTestDataHelper,MindDataTestTensorWriteUInt)172 TEST_F(MindDataTestDataHelper, MindDataTestTensorWriteUInt) {
173 // create tensor
174 std::vector<uint8_t> y = {1, 2, 3, 4};
175 std::shared_ptr<Tensor> t;
176 Tensor::CreateFromVector(y, &t);
177 uint8_t o;
178 t->GetItemAt<uint8_t>(&o, {0, 0});
179 MS_LOG(INFO) << "before op :" << std::to_string(o) << ".";
180
181 // create buffer using system mempool
182 DataHelper dh;
183 void *data = malloc(t->SizeInBytes());
184 if (data == nullptr) {
185 MS_LOG(ERROR) << "malloc failed";
186 ASSERT_TRUE(false);
187 }
188 auto bytes_copied = dh.DumpData(t->GetBuffer(), t->SizeInBytes(), data, t->SizeInBytes());
189 if (bytes_copied != t->SizeInBytes()) {
190 EXPECT_TRUE(false);
191 }
192 t->GetItemAt<uint8_t>(&o, {});
193 MS_LOG(INFO) << "after op :" << std::to_string(o) << ".";
194
195 uint8_t *array = static_cast<uint8_t *>(data);
196 if (array[0] != 1) { EXPECT_TRUE(false); }
197 if (array[1] != 2) { EXPECT_TRUE(false); }
198 if (array[2] != 3) { EXPECT_TRUE(false); }
199 if (array[3] != 4) { EXPECT_TRUE(false); }
200 std::free(data);
201 }
202
203
204