1 /**
2 * Copyright 2019 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "minddata/mindrecord/include/shard_statistics.h"
18 #include "pybind11/pybind11.h"
19
20 using mindspore::LogStream;
21 using mindspore::ExceptionType::NoExceptionType;
22 using mindspore::MsLogLevel::ERROR;
23
24 namespace mindspore {
25 namespace mindrecord {
Build(std::string desc,const json & statistics)26 std::shared_ptr<Statistics> Statistics::Build(std::string desc, const json &statistics) {
27 // validate check
28 if (!Validate(statistics)) {
29 return nullptr;
30 }
31 Statistics object_statistics;
32 object_statistics.desc_ = std::move(desc);
33 object_statistics.statistics_ = statistics;
34 object_statistics.statistics_id_ = -1;
35 return std::make_shared<Statistics>(object_statistics);
36 }
37
GetDesc() const38 std::string Statistics::GetDesc() const { return desc_; }
39
GetStatistics() const40 json Statistics::GetStatistics() const {
41 json str_statistics;
42 str_statistics["desc"] = desc_;
43 str_statistics["statistics"] = statistics_;
44 return str_statistics;
45 }
46
SetStatisticsID(int64_t id)47 void Statistics::SetStatisticsID(int64_t id) { statistics_id_ = id; }
48
GetStatisticsID() const49 int64_t Statistics::GetStatisticsID() const { return statistics_id_; }
50
Validate(const json & statistics)51 bool Statistics::Validate(const json &statistics) {
52 if (statistics.size() != kInt1) {
53 MS_LOG(ERROR) << "Invalid data, 'statistics' is empty.";
54 return false;
55 }
56 if (statistics.find("level") == statistics.end()) {
57 MS_LOG(ERROR) << "Invalid data, 'level' object can not found in statistic";
58 return false;
59 }
60 return LevelRecursive(statistics["level"]);
61 }
62
LevelRecursive(json level)63 bool Statistics::LevelRecursive(json level) {
64 bool ini = true;
65 for (json::iterator it = level.begin(); it != level.end(); ++it) {
66 json a = it.value();
67 if (a.size() == kInt2) {
68 if ((a.find("key") == a.end()) || (a.find("count") == a.end())) {
69 MS_LOG(ERROR) << "Invalid data, the node field is 2, but 'key'/'count' object does not existed";
70 return false;
71 }
72 } else if (a.size() == kInt3) {
73 if ((a.find("key") == a.end()) || (a.find("count") == a.end()) || a.find("level") == a.end()) {
74 MS_LOG(ERROR) << "Invalid data, the node field is 3, but 'key'/'count'/'level' object does not existed";
75 return false;
76 } else {
77 ini = LevelRecursive(a.at("level"));
78 }
79 } else {
80 MS_LOG(ERROR) << "Invalid data, the node field is not equal to 2 or 3";
81 return false;
82 }
83 }
84 return ini;
85 }
86
operator ==(const Statistics & b) const87 bool Statistics::operator==(const Statistics &b) const {
88 if (this->GetStatistics() != b.GetStatistics()) {
89 return false;
90 }
91 return true;
92 }
93 } // namespace mindrecord
94 } // namespace mindspore
95