1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "minddata/dataset/util/json_helper.h"
17
18 #include <nlohmann/json.hpp>
19
20 #include "minddata/dataset/util/log_adapter.h"
21 #include "minddata/dataset/util/path.h"
22 #include "minddata/dataset/util/status.h"
23
24 namespace mindspore {
25 namespace dataset {
26 // Create a numbered json file from image folder
CreateAlbum(const std::string & in_dir,const std::string & out_dir)27 Status JsonHelper::CreateAlbum(const std::string &in_dir, const std::string &out_dir) {
28 // in check
29 Path base_dir = Path(in_dir);
30 RETURN_IF_NOT_OK(RealPath(in_dir));
31 if (!base_dir.IsDirectory() || !base_dir.Exists()) {
32 RETURN_STATUS_UNEXPECTED("Input dir is not a directory or doesn't exist");
33 }
34 // check if output_dir exists and create it if it does not exist
35 Path target_dir = Path(out_dir);
36 RETURN_IF_NOT_OK(target_dir.CreateDirectory());
37
38 // iterate over in dir and create json for all images
39 uint64_t index = 0;
40 auto dir_it = Path::DirIterator::OpenDirectory(&base_dir);
41 while (dir_it->HasNext()) {
42 Path v = dir_it->Next();
43 // check if found file fits image extension
44
45 // create json file in output dir with the path
46 std::string out_file = out_dir + "/" + std::to_string(index) + ".json";
47 RETURN_IF_NOT_OK(UpdateValue(out_file, "image", v.ToString(), out_file));
48 index++;
49 }
50 return Status::OK();
51 }
52
RealPath(const std::string & path)53 Status JsonHelper::RealPath(const std::string &path) {
54 std::string real_path;
55 RETURN_IF_NOT_OK(Path::RealPath(path, real_path));
56 return Status::OK();
57 }
58
59 // A print method typically used for debugging
Print(std::ostream & out) const60 void JsonHelper::Print(std::ostream &out) const {
61 out << " Data Helper"
62 << "\n";
63 }
64
UpdateArray(const std::string & in_file,const std::string & key,const std::vector<std::string> & value,const std::string & out_file)65 Status JsonHelper::UpdateArray(const std::string &in_file, const std::string &key,
66 const std::vector<std::string> &value, const std::string &out_file) {
67 try {
68 Path in = Path(in_file);
69 nlohmann::json js;
70 if (in.Exists()) {
71 RETURN_IF_NOT_OK(RealPath(in_file));
72 try {
73 std::ifstream in_stream(in_file);
74 MS_LOG(INFO) << "Filename: " << in_file << ".";
75 in_stream >> js;
76 in_stream.close();
77 } catch (const std::exception &err) {
78 RETURN_STATUS_UNEXPECTED("Invalid file, failed to open json file: " + in_file +
79 ", please delete it and try again!");
80 }
81 }
82 js[key] = value;
83 MS_LOG(INFO) << "Write outfile is: " << js << ".";
84 if (out_file == "") {
85 std::ofstream o(in_file, std::ofstream::trunc);
86 o << js;
87 o.close();
88 } else {
89 std::ofstream o(out_file, std::ofstream::trunc);
90 o << js;
91 o.close();
92 }
93 }
94 // Catch any exception and convert to Status return code
95 catch (const std::exception &err) {
96 RETURN_STATUS_UNEXPECTED("Update json failed ");
97 }
98 return Status::OK();
99 }
100
RemoveKey(const std::string & in_file,const std::string & key,const std::string & out_file)101 Status JsonHelper::RemoveKey(const std::string &in_file, const std::string &key, const std::string &out_file) {
102 try {
103 Path in = Path(in_file);
104 nlohmann::json js;
105 if (in.Exists()) {
106 RETURN_IF_NOT_OK(RealPath(in_file));
107 try {
108 std::ifstream in_stream(in_file);
109 MS_LOG(INFO) << "Filename: " << in_file << ".";
110 in_stream >> js;
111 in_stream.close();
112 } catch (const std::exception &err) {
113 RETURN_STATUS_UNEXPECTED("Invalid file, failed to open json file: " + in_file +
114 ", please delete it and try again!");
115 }
116 }
117 (void)js.erase(key);
118 MS_LOG(INFO) << "Write outfile is: " << js << ".";
119 if (out_file == "") {
120 std::ofstream o(in_file, std::ofstream::trunc);
121 o << js;
122 o.close();
123 } else {
124 std::ofstream o(out_file, std::ofstream::trunc);
125 o << js;
126 o.close();
127 }
128 }
129 // Catch any exception and convert to Status return code
130 catch (const std::exception &err) {
131 RETURN_STATUS_UNEXPECTED("Update json failed ");
132 }
133 return Status::OK();
134 }
135
DumpData(const unsigned char * tensor_addr,const size_t & tensor_size,void * addr,const size_t & buffer_size)136 size_t JsonHelper::DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr,
137 const size_t &buffer_size) {
138 // write to address, input order is: destination, source
139 errno_t ret = memcpy_s(addr, buffer_size, tensor_addr, tensor_size);
140 if (ret != 0) {
141 // memcpy failed
142 MS_LOG(ERROR) << "memcpy tensor memory failed"
143 << ".";
144 return 0; // amount of data copied is 0, error
145 }
146 return tensor_size;
147 }
148 } // namespace dataset
149 } // namespace mindspore
150