/** * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "common.h" #include #include #include #include #include "minddata/dataset/core/client.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/pybind_support.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/engine/datasetops/batch_op.h" #include "minddata/dataset/engine/datasetops/repeat_op.h" #include "minddata/dataset/engine/datasetops/source/tf_reader_op.h" namespace UT { #ifdef __cplusplus #if __cplusplus extern "C" { #endif #endif void DatasetOpTesting::SetUp() { std::string install_home = "data/dataset"; datasets_root_path_ = install_home; mindrecord_root_path_ = "data/mindrecord"; } std::vector DatasetOpTesting::ToTensorShapeVec( const std::vector> &v) { std::vector ret_v; std::transform(v.begin(), v.end(), std::back_inserter(ret_v), [](const auto &s) { return mindspore::dataset::TensorShape(s); }); return ret_v; } std::vector DatasetOpTesting::ToDETypes(const std::vector &t) { std::vector ret_t; std::transform(t.begin(), t.end(), std::back_inserter(ret_t), [](const mindspore::DataType &t) { return mindspore::dataset::MSTypeToDEType(static_cast(t)); }); return ret_t; } // Function to read a file into an MSTensor // Note: This provides the analogous support for DETensor's CreateFromFile. mindspore::MSTensor DatasetOpTesting::ReadFileToTensor(const std::string &file) { if (file.empty()) { MS_LOG(ERROR) << "Pointer file is nullptr; return an empty Tensor."; return mindspore::MSTensor(); } std::ifstream ifs(file); if (!ifs.good()) { MS_LOG(ERROR) << "File: " << file << " does not exist; return an empty Tensor."; return mindspore::MSTensor(); } if (!ifs.is_open()) { MS_LOG(ERROR) << "File: " << file << " open failed; return an empty Tensor."; return mindspore::MSTensor(); } ifs.seekg(0, std::ios::end); size_t size = ifs.tellg(); mindspore::MSTensor buf("file", mindspore::DataType::kNumberTypeUInt8, {static_cast(size)}, nullptr, size); ifs.seekg(0, std::ios::beg); ifs.read(reinterpret_cast(buf.MutableData()), size); ifs.close(); return buf; } // Helper function to create a batch op std::shared_ptr DatasetOpTesting::Batch(int32_t batch_size, bool drop, mindspore::dataset::PadInfo pad_map) { /* std::shared_ptr cfg = mindspore::dataset::GlobalContext::config_manager(); int32_t num_workers = cfg->num_parallel_workers(); int32_t op_connector_size = cfg->op_connector_size(); std::vector output_columns = {}; std::vector input_columns = {}; mindspore::dataset::py::function batch_size_func; mindspore::dataset::py::function batch_map_func; bool pad = false; if (!pad_map.empty()) { pad = true; } std::shared_ptr op = std::make_shared(batch_size, drop, pad, op_connector_size, num_workers, input_columns, output_columns, batch_size_func, batch_map_func, pad_map); return op; */ Status rc; std::shared_ptr op; rc = mindspore::dataset::BatchOp::Builder(batch_size).SetDrop(drop).SetPaddingMap(pad_map).Build(&op); EXPECT_TRUE(rc.IsOk()); return std::move(op); } std::shared_ptr DatasetOpTesting::Repeat(int repeat_cnt) { std::shared_ptr op = std::make_shared(repeat_cnt); return std::move(op); } std::shared_ptr DatasetOpTesting::TFReader(std::string file, int num_works) { std::shared_ptr config_manager = mindspore::dataset::GlobalContext::config_manager(); auto op_connector_size = config_manager->op_connector_size(); auto worker_connector_size = config_manager->worker_connector_size(); std::vector columns_to_load = {}; std::vector files = {file}; std::shared_ptr so = std::make_shared( num_works, worker_connector_size, 0, files, std::make_unique(), op_connector_size, columns_to_load, false, 1, 0, false); (void)so->Init(); return std::move(so); } std::shared_ptr DatasetOpTesting::Build( std::vector> ops) { std::shared_ptr tree = std::make_shared(); for (int i = 0; i < ops.size(); i++) { tree->AssociateNode(ops[i]); if (i > 0) { ops[i]->AddChild(std::move(ops[i - 1])); } if (i == ops.size() - 1) { tree->AssignRoot(ops[i]); } } return std::move(tree); } #ifdef __cplusplus #if __cplusplus } #endif #endif } // namespace UT