1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "components/variations/variations_seed_simulator.h"
6
7 #include <map>
8
9 #include "base/strings/stringprintf.h"
10 #include "components/variations/processed_study.h"
11 #include "components/variations/proto/study.pb.h"
12 #include "components/variations/variations_associated_data.h"
13 #include "testing/gtest/include/gtest/gtest.h"
14
15 namespace chrome_variations {
16
17 namespace {
18
19 // An implementation of EntropyProvider that always returns a specific entropy
20 // value, regardless of field trial.
21 class TestEntropyProvider : public base::FieldTrial::EntropyProvider {
22 public:
TestEntropyProvider(double entropy_value)23 explicit TestEntropyProvider(double entropy_value)
24 : entropy_value_(entropy_value) {}
~TestEntropyProvider()25 virtual ~TestEntropyProvider() {}
26
27 // base::FieldTrial::EntropyProvider implementation:
GetEntropyForTrial(const std::string & trial_name,uint32 randomization_seed) const28 virtual double GetEntropyForTrial(const std::string& trial_name,
29 uint32 randomization_seed) const OVERRIDE {
30 return entropy_value_;
31 }
32
33 private:
34 const double entropy_value_;
35
36 DISALLOW_COPY_AND_ASSIGN(TestEntropyProvider);
37 };
38
39 // Creates and activates a single-group field trial with name |trial_name| and
40 // group |group_name| and variations |params| (if not NULL).
CreateTrial(const std::string & trial_name,const std::string & group_name,const std::map<std::string,std::string> * params)41 void CreateTrial(const std::string& trial_name,
42 const std::string& group_name,
43 const std::map<std::string, std::string>* params) {
44 base::FieldTrialList::CreateFieldTrial(trial_name, group_name);
45 if (params != NULL)
46 AssociateVariationParams(trial_name, group_name, *params);
47 base::FieldTrialList::FindFullName(trial_name);
48 }
49
50 // Creates a study with the given |study_name| and |consistency|.
CreateStudy(const std::string & study_name,Study_Consistency consistency)51 Study CreateStudy(const std::string& study_name,
52 Study_Consistency consistency) {
53 Study study;
54 study.set_name(study_name);
55 study.set_consistency(consistency);
56 return study;
57 }
58
59 // Adds an experiment to |study| with the specified |experiment_name| and
60 // |probability| values and sets it as the study's default experiment.
AddExperiment(const std::string & experiment_name,int probability,Study * study)61 Study_Experiment* AddExperiment(const std::string& experiment_name,
62 int probability,
63 Study* study) {
64 Study_Experiment* experiment = study->add_experiment();
65 experiment->set_name(experiment_name);
66 experiment->set_probability_weight(probability);
67 study->set_default_experiment_name(experiment_name);
68 return experiment;
69 }
70
71 // Add an experiment param with |param_name| and |param_value| to |experiment|.
AddExperimentParam(const std::string & param_name,const std::string & param_value,Study_Experiment * experiment)72 Study_Experiment_Param* AddExperimentParam(const std::string& param_name,
73 const std::string& param_value,
74 Study_Experiment* experiment) {
75 Study_Experiment_Param* param = experiment->add_param();
76 param->set_name(param_name);
77 param->set_value(param_value);
78 return param;
79 }
80
81 } // namespace
82
83 class VariationsSeedSimulatorTest : public ::testing::Test {
84 public:
VariationsSeedSimulatorTest()85 VariationsSeedSimulatorTest() : field_trial_list_(NULL) {
86 }
87
~VariationsSeedSimulatorTest()88 virtual ~VariationsSeedSimulatorTest() {
89 // Ensure that the maps are cleared between tests, since they are stored as
90 // process singletons.
91 testing::ClearAllVariationIDs();
92 testing::ClearAllVariationParams();
93 }
94
95 // Uses a VariationsSeedSimulator to simulate the differences between
96 // |studies| and the current field trial state.
SimulateDifferences(const std::vector<ProcessedStudy> & studies)97 VariationsSeedSimulator::Result SimulateDifferences(
98 const std::vector<ProcessedStudy>& studies) {
99 TestEntropyProvider provider(0.5);
100 VariationsSeedSimulator seed_simulator(provider);
101 return seed_simulator.ComputeDifferences(studies);
102 }
103
104 // Simulates the differences between |study| and the current field trial
105 // state, returning a string like "1 2 3", where 1 is the number of regular
106 // group changes, 2 is the number of "kill best effort" group changes and 3
107 // is the number of "kill critical" group changes.
SimulateStudyDifferences(const Study * study)108 std::string SimulateStudyDifferences(const Study* study) {
109 std::vector<ProcessedStudy> studies;
110 if (!ProcessedStudy::ValidateAndAppendStudy(study, false, &studies))
111 return "invalid study";
112 return ConvertSimulationResultToString(SimulateDifferences(studies));
113
114 }
115
116 // Simulates the differences between expired |study| and the current field
117 // trial state, returning a string like "1 2 3", where 1 is the number of
118 // regular group changes, 2 is the number of "kill best effort" group changes
119 // and 3 is the number of "kill critical" group changes.
SimulateStudyDifferencesExpired(const Study * study)120 std::string SimulateStudyDifferencesExpired(const Study* study) {
121 std::vector<ProcessedStudy> studies;
122 if (!ProcessedStudy::ValidateAndAppendStudy(study, true, &studies))
123 return "invalid study";
124 if (!studies[0].is_expired())
125 return "not expired";
126 return ConvertSimulationResultToString(SimulateDifferences(studies));
127 }
128
129 // Formats |result| as a string with format "1 2 3", where 1 is the number of
130 // regular group changes, 2 is the number of "kill best effort" group changes
131 // and 3 is the number of "kill critical" group changes.
ConvertSimulationResultToString(const VariationsSeedSimulator::Result & result)132 std::string ConvertSimulationResultToString(
133 const VariationsSeedSimulator::Result& result) {
134 return base::StringPrintf("%d %d %d",
135 result.normal_group_change_count,
136 result.kill_best_effort_group_change_count,
137 result.kill_critical_group_change_count);
138 }
139
140 private:
141 base::FieldTrialList field_trial_list_;
142
143 DISALLOW_COPY_AND_ASSIGN(VariationsSeedSimulatorTest);
144 };
145
TEST_F(VariationsSeedSimulatorTest,PermanentNoChanges)146 TEST_F(VariationsSeedSimulatorTest, PermanentNoChanges) {
147 CreateTrial("A", "B", NULL);
148
149 std::vector<ProcessedStudy> processed_studies;
150 Study study = CreateStudy("A", Study_Consistency_PERMANENT);
151 Study_Experiment* experiment = AddExperiment("B", 100, &study);
152
153 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
154
155 experiment->set_type(Study_Experiment_Type_NORMAL);
156 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
157 experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
158 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
159 experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
160 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
161 experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
162 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
163 }
164
TEST_F(VariationsSeedSimulatorTest,PermanentGroupChange)165 TEST_F(VariationsSeedSimulatorTest, PermanentGroupChange) {
166 CreateTrial("A", "B", NULL);
167
168 Study study = CreateStudy("A", Study_Consistency_PERMANENT);
169 Study_Experiment* experiment = AddExperiment("C", 100, &study);
170
171 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
172
173 // Changing "C" group type should not affect the type of change. (Since the
174 // type is evaluated for the "old" group.)
175 experiment->set_type(Study_Experiment_Type_NORMAL);
176 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
177 experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
178 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
179 experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
180 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
181 experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
182 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
183 }
184
TEST_F(VariationsSeedSimulatorTest,PermanentExpired)185 TEST_F(VariationsSeedSimulatorTest, PermanentExpired) {
186 CreateTrial("A", "B", NULL);
187
188 Study study = CreateStudy("A", Study_Consistency_PERMANENT);
189 Study_Experiment* experiment = AddExperiment("B", 1, &study);
190 AddExperiment("C", 0, &study);
191
192 // There should be a difference because the study is expired, which should
193 // result in the default group "C" being chosen.
194 EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
195
196 experiment->set_type(Study_Experiment_Type_NORMAL);
197 EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
198 experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
199 EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
200 experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
201 EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
202 experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
203 EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
204 }
205
TEST_F(VariationsSeedSimulatorTest,SessionRandomized)206 TEST_F(VariationsSeedSimulatorTest, SessionRandomized) {
207 CreateTrial("A", "B", NULL);
208
209 Study study = CreateStudy("A", Study_Consistency_SESSION);
210 Study_Experiment* experiment = AddExperiment("B", 1, &study);
211 AddExperiment("C", 1, &study);
212 AddExperiment("D", 1, &study);
213
214 // There should be no differences, since a session randomized study can result
215 // in any of the groups being chosen on startup.
216 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
217
218 experiment->set_type(Study_Experiment_Type_NORMAL);
219 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
220 experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
221 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
222 experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
223 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
224 experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
225 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
226 }
227
TEST_F(VariationsSeedSimulatorTest,SessionRandomizedGroupRemoved)228 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupRemoved) {
229 CreateTrial("A", "B", NULL);
230
231 Study study = CreateStudy("A", Study_Consistency_SESSION);
232 AddExperiment("C", 1, &study);
233 AddExperiment("D", 1, &study);
234
235 // There should be a difference since there is no group "B" in the new config.
236 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
237 }
238
TEST_F(VariationsSeedSimulatorTest,SessionRandomizedGroupProbabilityZero)239 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupProbabilityZero) {
240 CreateTrial("A", "B", NULL);
241
242 Study study = CreateStudy("A", Study_Consistency_SESSION);
243 Study_Experiment* experiment = AddExperiment("B", 0, &study);
244 AddExperiment("C", 1, &study);
245 AddExperiment("D", 1, &study);
246
247 // There should be a difference since group "B" has probability 0.
248 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
249
250 experiment->set_type(Study_Experiment_Type_NORMAL);
251 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
252 experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
253 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
254 experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
255 EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
256 experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
257 EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
258 }
259
TEST_F(VariationsSeedSimulatorTest,SessionRandomizedExpired)260 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedExpired) {
261 CreateTrial("A", "B", NULL);
262
263 Study study = CreateStudy("A", Study_Consistency_SESSION);
264 Study_Experiment* experiment = AddExperiment("B", 1, &study);
265 AddExperiment("C", 1, &study);
266 AddExperiment("D", 1, &study);
267
268 // There should be a difference because the study is expired, which should
269 // result in the default group "D" being chosen.
270 EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
271
272 experiment->set_type(Study_Experiment_Type_NORMAL);
273 EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
274 experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
275 EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
276 experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
277 EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
278 experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
279 EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
280 }
281
TEST_F(VariationsSeedSimulatorTest,ParamsUnchanged)282 TEST_F(VariationsSeedSimulatorTest, ParamsUnchanged) {
283 std::map<std::string, std::string> params;
284 params["p1"] = "x";
285 params["p2"] = "y";
286 params["p3"] = "z";
287 CreateTrial("A", "B", ¶ms);
288
289 std::vector<ProcessedStudy> processed_studies;
290 Study study = CreateStudy("A", Study_Consistency_PERMANENT);
291 Study_Experiment* experiment = AddExperiment("B", 100, &study);
292 AddExperimentParam("p2", "y", experiment);
293 AddExperimentParam("p1", "x", experiment);
294 AddExperimentParam("p3", "z", experiment);
295
296 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
297
298 experiment->set_type(Study_Experiment_Type_NORMAL);
299 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
300 experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
301 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
302 experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
303 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
304 experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
305 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
306 }
307
TEST_F(VariationsSeedSimulatorTest,ParamsChanged)308 TEST_F(VariationsSeedSimulatorTest, ParamsChanged) {
309 std::map<std::string, std::string> params;
310 params["p1"] = "x";
311 params["p2"] = "y";
312 params["p3"] = "z";
313 CreateTrial("A", "B", ¶ms);
314
315 std::vector<ProcessedStudy> processed_studies;
316 Study study = CreateStudy("A", Study_Consistency_PERMANENT);
317 Study_Experiment* experiment = AddExperiment("B", 100, &study);
318 AddExperimentParam("p2", "test", experiment);
319 AddExperimentParam("p1", "x", experiment);
320 AddExperimentParam("p3", "z", experiment);
321
322 // The param lists differ.
323 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
324
325 experiment->set_type(Study_Experiment_Type_NORMAL);
326 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
327 experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
328 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
329 experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
330 EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
331 experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
332 EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
333 }
334
TEST_F(VariationsSeedSimulatorTest,ParamsRemoved)335 TEST_F(VariationsSeedSimulatorTest, ParamsRemoved) {
336 std::map<std::string, std::string> params;
337 params["p1"] = "x";
338 params["p2"] = "y";
339 params["p3"] = "z";
340 CreateTrial("A", "B", ¶ms);
341
342 std::vector<ProcessedStudy> processed_studies;
343 Study study = CreateStudy("A", Study_Consistency_PERMANENT);
344 Study_Experiment* experiment = AddExperiment("B", 100, &study);
345
346 // The current group has params, but the new config doesn't have any.
347 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
348
349 experiment->set_type(Study_Experiment_Type_NORMAL);
350 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
351 experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
352 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
353 experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
354 EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
355 experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
356 EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
357 }
358
TEST_F(VariationsSeedSimulatorTest,ParamsAdded)359 TEST_F(VariationsSeedSimulatorTest, ParamsAdded) {
360 CreateTrial("A", "B", NULL);
361
362 std::vector<ProcessedStudy> processed_studies;
363 Study study = CreateStudy("A", Study_Consistency_PERMANENT);
364 Study_Experiment* experiment = AddExperiment("B", 100, &study);
365 AddExperimentParam("p2", "y", experiment);
366 AddExperimentParam("p1", "x", experiment);
367 AddExperimentParam("p3", "z", experiment);
368
369 // The current group has no params, but the config has added some.
370 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
371
372 experiment->set_type(Study_Experiment_Type_NORMAL);
373 EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
374 experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
375 EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
376 experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
377 EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
378 experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
379 EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
380 }
381
382 } // namespace chrome_variations
383