1 /*
2 *
3 * Copyright 2015-2016 gRPC authors.
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 *
17 */
18
19 #include <fstream>
20 #include <iostream>
21 #include <memory>
22 #include <set>
23
24 #include <grpcpp/impl/codegen/config_protobuf.h>
25
26 #include <gflags/gflags.h>
27 #include <grpc/support/log.h>
28
29 #include "test/cpp/qps/benchmark_config.h"
30 #include "test/cpp/qps/driver.h"
31 #include "test/cpp/qps/parse_json.h"
32 #include "test/cpp/qps/report.h"
33 #include "test/cpp/qps/server.h"
34 #include "test/cpp/util/test_config.h"
35 #include "test/cpp/util/test_credentials_provider.h"
36
37 DEFINE_string(scenarios_file, "",
38 "JSON file containing an array of Scenario objects");
39 DEFINE_string(scenarios_json, "",
40 "JSON string containing an array of Scenario objects");
41 DEFINE_bool(quit, false, "Quit the workers");
42 DEFINE_string(search_param, "",
43 "The parameter, whose value is to be searched for to achieve "
44 "targeted cpu load. For now, we have 'offered_load'. Later, "
45 "'num_channels', 'num_outstanding_requests', etc. shall be "
46 "added.");
47 DEFINE_double(
48 initial_search_value, 0.0,
49 "initial parameter value to start the search with (i.e. lower bound)");
50 DEFINE_double(targeted_cpu_load, 70.0,
51 "Targeted cpu load (unit: %, range [0,100])");
52 DEFINE_double(stride, 1,
53 "Defines each stride of the search. The larger the stride is, "
54 "the coarser the result will be, but will also be faster.");
55 DEFINE_double(error_tolerance, 0.01,
56 "Defines threshold for stopping the search. When current search "
57 "range is narrower than the error_tolerance computed range, we "
58 "stop the search.");
59
60 DEFINE_string(qps_server_target_override, "",
61 "Override QPS server target to configure in client configs."
62 "Only applicable if there is a single benchmark server.");
63
64 DEFINE_string(json_file_out, "", "File to write the JSON output to.");
65
66 DEFINE_string(credential_type, grpc::testing::kInsecureCredentialsType,
67 "Credential type for communication with workers");
68 DEFINE_bool(run_inproc, false, "Perform an in-process transport test");
69 DEFINE_int32(
70 median_latency_collection_interval_millis, 0,
71 "Specifies the period between gathering latency medians in "
72 "milliseconds. The medians will be logged out on the client at the "
73 "end of the benchmark run. If 0, this periodic collection is disabled.");
74
75 namespace grpc {
76 namespace testing {
77
RunAndReport(const Scenario & scenario,bool * success)78 static std::unique_ptr<ScenarioResult> RunAndReport(const Scenario& scenario,
79 bool* success) {
80 std::cerr << "RUNNING SCENARIO: " << scenario.name() << "\n";
81 auto result = RunScenario(
82 scenario.client_config(), scenario.num_clients(),
83 scenario.server_config(), scenario.num_servers(),
84 scenario.warmup_seconds(), scenario.benchmark_seconds(),
85 !FLAGS_run_inproc ? scenario.spawn_local_worker_count() : -2,
86 FLAGS_qps_server_target_override, FLAGS_credential_type, FLAGS_run_inproc,
87 FLAGS_median_latency_collection_interval_millis);
88
89 // Amend the result with scenario config. Eventually we should adjust
90 // RunScenario contract so we don't need to touch the result here.
91 result->mutable_scenario()->CopyFrom(scenario);
92
93 GetReporter()->ReportQPS(*result);
94 GetReporter()->ReportQPSPerCore(*result);
95 GetReporter()->ReportLatency(*result);
96 GetReporter()->ReportTimes(*result);
97 GetReporter()->ReportCpuUsage(*result);
98 GetReporter()->ReportPollCount(*result);
99 GetReporter()->ReportQueriesPerCpuSec(*result);
100
101 for (int i = 0; *success && i < result->client_success_size(); i++) {
102 *success = result->client_success(i);
103 }
104 for (int i = 0; *success && i < result->server_success_size(); i++) {
105 *success = result->server_success(i);
106 }
107
108 if (FLAGS_json_file_out != "") {
109 std::ofstream json_outfile;
110 json_outfile.open(FLAGS_json_file_out);
111 json_outfile << "{\"qps\": " << result->summary().qps() << "}\n";
112 json_outfile.close();
113 }
114
115 return result;
116 }
117
GetCpuLoad(Scenario * scenario,double offered_load,bool * success)118 static double GetCpuLoad(Scenario* scenario, double offered_load,
119 bool* success) {
120 scenario->mutable_client_config()
121 ->mutable_load_params()
122 ->mutable_poisson()
123 ->set_offered_load(offered_load);
124 auto result = RunAndReport(*scenario, success);
125 return result->summary().server_cpu_usage();
126 }
127
BinarySearch(Scenario * scenario,double targeted_cpu_load,double low,double high,bool * success)128 static double BinarySearch(Scenario* scenario, double targeted_cpu_load,
129 double low, double high, bool* success) {
130 while (low <= high * (1 - FLAGS_error_tolerance)) {
131 double mid = low + (high - low) / 2;
132 double current_cpu_load = GetCpuLoad(scenario, mid, success);
133 gpr_log(GPR_DEBUG, "Binary Search: current_offered_load %.0f", mid);
134 if (!*success) {
135 gpr_log(GPR_ERROR, "Client/Server Failure");
136 break;
137 }
138 if (targeted_cpu_load <= current_cpu_load) {
139 high = mid - FLAGS_stride;
140 } else {
141 low = mid + FLAGS_stride;
142 }
143 }
144
145 return low;
146 }
147
SearchOfferedLoad(double initial_offered_load,double targeted_cpu_load,Scenario * scenario,bool * success)148 static double SearchOfferedLoad(double initial_offered_load,
149 double targeted_cpu_load, Scenario* scenario,
150 bool* success) {
151 std::cerr << "RUNNING SCENARIO: " << scenario->name() << "\n";
152 double current_offered_load = initial_offered_load;
153 double current_cpu_load = GetCpuLoad(scenario, current_offered_load, success);
154 if (current_cpu_load > targeted_cpu_load) {
155 gpr_log(GPR_ERROR, "Initial offered load too high");
156 return -1;
157 }
158
159 while (*success && (current_cpu_load < targeted_cpu_load)) {
160 current_offered_load *= 2;
161 current_cpu_load = GetCpuLoad(scenario, current_offered_load, success);
162 gpr_log(GPR_DEBUG, "Binary Search: current_offered_load %.0f",
163 current_offered_load);
164 }
165
166 double targeted_offered_load =
167 BinarySearch(scenario, targeted_cpu_load, current_offered_load / 2,
168 current_offered_load, success);
169
170 return targeted_offered_load;
171 }
172
QpsDriver()173 static bool QpsDriver() {
174 grpc::string json;
175
176 bool scfile = (FLAGS_scenarios_file != "");
177 bool scjson = (FLAGS_scenarios_json != "");
178 if ((!scfile && !scjson && !FLAGS_quit) ||
179 (scfile && (scjson || FLAGS_quit)) || (scjson && FLAGS_quit)) {
180 gpr_log(GPR_ERROR,
181 "Exactly one of --scenarios_file, --scenarios_json, "
182 "or --quit must be set");
183 abort();
184 }
185
186 if (scfile) {
187 // Read the json data from disk
188 FILE* json_file = fopen(FLAGS_scenarios_file.c_str(), "r");
189 GPR_ASSERT(json_file != nullptr);
190 fseek(json_file, 0, SEEK_END);
191 long len = ftell(json_file);
192 char* data = new char[len];
193 fseek(json_file, 0, SEEK_SET);
194 GPR_ASSERT(len == (long)fread(data, 1, len, json_file));
195 fclose(json_file);
196 json = grpc::string(data, data + len);
197 delete[] data;
198 } else if (scjson) {
199 json = FLAGS_scenarios_json.c_str();
200 } else if (FLAGS_quit) {
201 return RunQuit(FLAGS_credential_type);
202 }
203
204 // Parse into an array of scenarios
205 Scenarios scenarios;
206 ParseJson(json.c_str(), "grpc.testing.Scenarios", &scenarios);
207 bool success = true;
208
209 // Make sure that there is at least some valid scenario here
210 GPR_ASSERT(scenarios.scenarios_size() > 0);
211
212 for (int i = 0; i < scenarios.scenarios_size(); i++) {
213 if (FLAGS_search_param == "") {
214 const Scenario& scenario = scenarios.scenarios(i);
215 RunAndReport(scenario, &success);
216 } else {
217 if (FLAGS_search_param == "offered_load") {
218 Scenario* scenario = scenarios.mutable_scenarios(i);
219 double targeted_offered_load =
220 SearchOfferedLoad(FLAGS_initial_search_value,
221 FLAGS_targeted_cpu_load, scenario, &success);
222 gpr_log(GPR_INFO, "targeted_offered_load %f", targeted_offered_load);
223 GetCpuLoad(scenario, targeted_offered_load, &success);
224 } else {
225 gpr_log(GPR_ERROR, "Unimplemented search param");
226 }
227 }
228 }
229 return success;
230 }
231
232 } // namespace testing
233 } // namespace grpc
234
main(int argc,char ** argv)235 int main(int argc, char** argv) {
236 grpc::testing::InitTest(&argc, &argv, true);
237
238 bool ok = grpc::testing::QpsDriver();
239
240 return ok ? 0 : 1;
241 }
242