1 /**
2 * Copyright 2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "minddata/dataset/util/rdr.h"
17 #include "minddata/dataset/util/log_adapter.h"
18
19 namespace mindspore {
20 namespace dataset {
21 const int32_t kMdRdrRecordLimit = 10;
22
ToString()23 std::string MDChannelInfo::ToString() {
24 std::ostringstream ss;
25 ss << "have_sent: " << preprocess_batch_ << "; ";
26 ss << "host_queue: ";
27 for (uint32_t i = 0; i < batch_queue_.size(); i++) {
28 ss << batch_queue_.at(i);
29 if (i < batch_queue_.size() - 1) {
30 ss << ", ";
31 }
32 }
33
34 ss << "; push_start_time: ";
35 for (uint32_t i = 0; i < push_start_time_.size(); i++) {
36 ss << push_start_time_.at(i);
37 if (i < push_start_time_.size() - 1) {
38 ss << ", ";
39 }
40 }
41
42 ss << "; push_end_time: ";
43 for (uint32_t i = 0; i < push_end_time_.size(); i++) {
44 ss << push_end_time_.at(i);
45 if (i < push_end_time_.size() - 1) {
46 ss << ", ";
47 }
48 }
49 ss << ".";
50 return ss.str();
51 }
52
ToFormatString()53 std::string MDChannelInfo::ToFormatString() {
54 std::ostringstream ss;
55 // the output like below:
56 // channel_name: 29475464-f51b-11ee-b72b-8feb6783b0c3
57 // have_sent: 282;
58 // host_queue: 64, 64, 64, 63, 64, 64, 64, 63, 64, 64;
59 // device_queue: 99, 99, 99, 99, 98, 99, 97, 99, 98, 99;
60 // push_first_start_time -> push_first_end_time
61 // 2022-05-09-14:29:12.110.276 -> 2022-05-09-14:29:12.439.621
62 // push_start_time -> push_end_time
63 // -> 2022-05-09-14:31:00.603.866
64 // 2022-05-09-14:31:00.621.146 -> 2022-05-09-14:31:01.018.964
65 // 2022-05-09-14:31:01.043.705 -> 2022-05-09-14:31:01.396.650
66 // 2022-05-09-14:31:01.421.501 -> 2022-05-09-14:31:01.807.671
67 // 2022-05-09-14:31:01.828.931 -> 2022-05-09-14:31:02.179.945
68 // 2022-05-09-14:31:02.201.960 -> 2022-05-09-14:31:02.555.941
69 // 2022-05-09-14:31:02.584.413 -> 2022-05-09-14:31:02.943.839
70 // 2022-05-09-14:31:02.969.583 -> 2022-05-09-14:31:03.309.299
71 // 2022-05-09-14:31:03.337.607 -> 2022-05-09-14:31:03.684.034
72 // 2022-05-09-14:31:03.717.230 -> 2022-05-09-14:31:04.038.521
73 // 2022-05-09-14:31:04.064.571 ->
74 ss << "\n";
75 ss << "channel_name: " << channel_name_ << ";\n";
76 ss << "have_sent: " << preprocess_batch_ << ";\n";
77 ss << "host_queue: ";
78 for (uint32_t i = 0; i < batch_queue_.size(); i++) {
79 ss << batch_queue_.at(i);
80 if (i < batch_queue_.size() - 1) {
81 ss << ", ";
82 }
83 }
84 ss << ";\n";
85 ss << "device_queue: ";
86 for (uint32_t i = 0; i < device_queue_.size(); i++) {
87 ss << device_queue_.at(i);
88 if (i < device_queue_.size() - 1) {
89 ss << ", ";
90 }
91 }
92 ss << ";\n";
93 ss << " push_first_start_time -> push_first_end_time\n";
94 ss << push_first_start_time_ << " -> " << push_first_end_time_ << "\n";
95 ss << " push_start_time -> push_end_time\n";
96 if (!push_start_time_.empty()) {
97 if (!push_end_time_.empty()) {
98 // start_time[0] bigger than end_time[0]
99 uint32_t end_time_index = 0;
100 if (push_start_time_.at(0) > push_end_time_.at(0)) {
101 ss << " -> " << push_end_time_.at(end_time_index) << "\n";
102 end_time_index = 1;
103 }
104 for (uint32_t i = 0; i < push_start_time_.size(); i++, end_time_index++) {
105 ss << push_start_time_.at(i) << " -> ";
106 if (end_time_index < push_end_time_.size()) {
107 ss << push_end_time_.at(end_time_index);
108 }
109 ss << "\n";
110 }
111 } else {
112 ss << push_start_time_.at(0) << " -> \n"; // only one start time without end time
113 }
114 }
115 ss << "For more details, please refer to the FAQ at "
116 << "https://www.mindspore.cn/docs/en/master/faq/data_processing.html.";
117 return ss.str();
118 }
119
RecordBatchQueue(int64_t batch_queue_size)120 Status MDChannelInfo::RecordBatchQueue(int64_t batch_queue_size) {
121 if (batch_queue_.size() == kMdRdrRecordLimit) {
122 batch_queue_.pop_front();
123 }
124 batch_queue_.push_back(batch_queue_size);
125 return Status::OK();
126 }
127
RecordDeviceQueue(int64_t device_queue_size)128 Status MDChannelInfo::RecordDeviceQueue(int64_t device_queue_size) {
129 if (device_queue_.size() == kMdRdrRecordLimit) {
130 device_queue_.pop_front();
131 }
132 device_queue_.push_back(device_queue_size);
133 return Status::OK();
134 }
135
RecordPreprocessBatch(int64_t preprocess_batch)136 Status MDChannelInfo::RecordPreprocessBatch(int64_t preprocess_batch) {
137 preprocess_batch_ = preprocess_batch;
138 return Status::OK();
139 }
140
RecordPushFirstStartTime()141 Status MDChannelInfo::RecordPushFirstStartTime() {
142 push_first_start_time_ = GetTimeString();
143 return Status::OK();
144 }
145
RecordPushFirstEndTime()146 Status MDChannelInfo::RecordPushFirstEndTime() {
147 push_first_end_time_ = GetTimeString();
148 return Status::OK();
149 }
150
RecordPushStartTime()151 Status MDChannelInfo::RecordPushStartTime() {
152 if (push_start_time_.size() == kMdRdrRecordLimit) {
153 push_start_time_.pop_front();
154 }
155 push_start_time_.push_back(GetTimeString());
156 return Status::OK();
157 }
158
RecordPushEndTime()159 Status MDChannelInfo::RecordPushEndTime() {
160 if (push_end_time_.size() == kMdRdrRecordLimit) {
161 push_end_time_.pop_front();
162 }
163 push_end_time_.push_back(GetTimeString());
164 return Status::OK();
165 }
166 } // namespace dataset
167 } // namespace mindspore
168