1 /** 2 * Copyright 2020 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_CONNECTOR_THROUGHPUT_H 18 #define MINDSPORE_CCSRC_MINDDATA_DATASET_CONNECTOR_THROUGHPUT_H 19 20 #include <vector> 21 #include <chrono> 22 #include <fstream> 23 #include <string> 24 #include <nlohmann/json.hpp> 25 #include "minddata/dataset/engine/perf/profiling.h" 26 #include "minddata/dataset/engine/perf/perf_data.h" 27 #include "minddata/dataset/engine/perf/cyclic_array.h" 28 #include "minddata/dataset/engine/datasetops/dataset_op.h" 29 #include "minddata/dataset/engine/execution_tree.h" 30 31 using json = nlohmann::json; 32 namespace mindspore { 33 namespace dataset { 34 // Connector throughput samples the output connector size of each op in the pipeline. 35 // For the description of the data structure see perf_data.h 36 // It support JSON serialization for external usage. 37 class ConnectorThroughput : public Sampling { 38 using OutRowCount = PerfData<CyclicArray<int64_t>>; 39 using Throughput = PerfData<CyclicArray<double>>; 40 using TimePoint = std::chrono::time_point<std::chrono::steady_clock>; 41 using TimeStamps = PerfData<CyclicArray<TimePoint>>; 42 43 public: 44 explicit ConnectorThroughput(ExecutionTree *tree, int64_t max_rows = 1000000) tree_(tree)45 : tree_(tree), 46 max_rows_(max_rows), 47 n_nodes_(InitNodes()), 48 out_row_count_table_(OutRowCount(max_rows_, n_nodes_)), 49 throughput_(Throughput(max_rows_, n_nodes_)), 50 timestamps_(TimeStamps(max_rows_, 1)) { 51 timestamps_.AddSample(std::vector<TimePoint>(1)); 52 out_row_count_table_.AddSample(std::vector<int64_t>(n_nodes_)); 53 } 54 55 /// \brief Destructor 56 ~ConnectorThroughput() = default; 57 58 // Driver function for connector size sampling. 59 // This function samples the connector size of every nodes within the ExecutionTree 60 Status Sample() override; 61 62 // Traverse the tree nodes and count them 63 int InitNodes(); 64 Name()65 std::string Name() const override { return name_; }; 66 67 // Save sampling data to file 68 // @return Status The status code returned 69 Status SaveToFile() override; 70 71 Status Init(const std::string &dir_path, const std::string &device_id) override; 72 73 json ParseOpInfo(const DatasetOp &node, const std::vector<double> &thr); 74 75 Status ChangeFileMode() override; 76 77 Status Analyze() override; 78 79 private: 80 ExecutionTree *tree_ = nullptr; // ExecutionTree pointer 81 int64_t max_rows_; 82 int32_t n_nodes_; 83 OutRowCount out_row_count_table_; 84 Throughput throughput_; 85 TimeStamps timestamps_; 86 std::string name_ = kConnectorThroughputSamplingName; 87 }; 88 89 } // namespace dataset 90 } // namespace mindspore 91 92 #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_CONNECTOR_THROUGHPUT_H 93