1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <gtest/gtest_prod.h> 20 21 #include <optional> 22 23 #include "ValueMetricProducer.h" 24 25 namespace android { 26 namespace os { 27 namespace statsd { 28 29 // TODO(b/185796344): don't use Value from FieldValue. 30 using ValueBases = std::vector<std::optional<Value>>; 31 class NumericValueMetricProducer : public ValueMetricProducer<Value, ValueBases> { 32 public: 33 NumericValueMetricProducer(const ConfigKey& key, const ValueMetric& valueMetric, 34 const uint64_t protoHash, const PullOptions& pullOptions, 35 const BucketOptions& bucketOptions, const WhatOptions& whatOptions, 36 const ConditionOptions& conditionOptions, 37 const StateOptions& stateOptions, 38 const ActivationOptions& activationOptions, 39 const GuardrailOptions& guardrailOptions); 40 41 // Process data pulled on bucket boundary. 42 void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& data, bool pullSuccess, 43 int64_t originalPullTimeNs) override; 44 getMetricType()45 inline MetricType getMetricType() const override { 46 return METRIC_TYPE_VALUE; 47 } 48 49 protected: 50 private: 51 void prepareFirstBucketLocked() override; 52 getConditionIdForMetric(const StatsdConfig & config,const int configIndex)53 inline optional<int64_t> getConditionIdForMetric(const StatsdConfig& config, 54 const int configIndex) const override { 55 const ValueMetric& metric = config.value_metric(configIndex); 56 return metric.has_condition() ? make_optional(metric.condition()) : nullopt; 57 } 58 getWhatAtomMatcherIdForMetric(const StatsdConfig & config,const int configIndex)59 inline int64_t getWhatAtomMatcherIdForMetric(const StatsdConfig& config, 60 const int configIndex) const override { 61 return config.value_metric(configIndex).what(); 62 } 63 getConditionLinksForMetric(const StatsdConfig & config,const int configIndex)64 inline ConditionLinks getConditionLinksForMetric(const StatsdConfig& config, 65 const int configIndex) const override { 66 return config.value_metric(configIndex).links(); 67 } 68 69 void onActiveStateChangedInternalLocked(const int64_t eventTimeNs) override; 70 71 // Only called when mIsActive and the event is NOT too late. 72 void onConditionChangedInternalLocked(const ConditionState oldCondition, 73 const ConditionState newCondition, 74 const int64_t eventTimeNs) override; 75 aggregatedValueToString(const Value & value)76 inline std::string aggregatedValueToString(const Value& value) const override { 77 return value.toString(); 78 } 79 80 // Mark the data as invalid. 81 void invalidateCurrentBucket(const int64_t dropTimeNs, const BucketDropReason reason) override; 82 83 // Reset diff base and mHasGlobalBase 84 void resetBase(); 85 86 // Calculate previous bucket end time based on current time. 87 int64_t calcPreviousBucketEndTime(const int64_t currentTimeNs); 88 multipleBucketsSkipped(const int64_t numBucketsForward)89 inline bool multipleBucketsSkipped(const int64_t numBucketsForward) const override { 90 return numBucketsForward > 1 && (isPulled() || mUseDiff); 91 } 92 93 // Process events retrieved from a pull. 94 void accumulateEvents(const std::vector<std::shared_ptr<LogEvent>>& allData, 95 int64_t originalPullTimeNs, int64_t eventElapsedTimeNs); 96 97 void closeCurrentBucket(const int64_t eventTimeNs, 98 const int64_t nextBucketStartTimeNs) override; 99 100 PastBucket<Value> buildPartialBucket(int64_t bucketEndTime, 101 std::vector<Interval>& intervals) override; 102 103 bool valuePassesThreshold(const Interval& interval) const; 104 105 Value getFinalValue(const Interval& interval) const; 106 107 void initNextSlicedBucket(int64_t nextBucketStartTimeNs) override; 108 109 void appendToFullBucket(const bool isFullBucketReached); 110 111 bool hitFullBucketGuardRailLocked(const MetricDimensionKey& newKey) const; 112 canSkipLogEventLocked(const MetricDimensionKey & eventKey,const bool condition,const int64_t eventTimeNs,const map<int,HashableDimensionKey> & statePrimaryKeys)113 inline bool canSkipLogEventLocked( 114 const MetricDimensionKey& eventKey, const bool condition, const int64_t eventTimeNs, 115 const map<int, HashableDimensionKey>& statePrimaryKeys) const override { 116 // For pushed metrics, can only skip if condition is false. 117 // For pulled metrics, can only skip if metric is not diffed and condition is false or 118 // unknown. 119 return (!isPulled() && !condition) || 120 (isPulled() && !mUseDiff && mCondition != ConditionState::kTrue); 121 } 122 123 bool aggregateFields(const int64_t eventTimeNs, const MetricDimensionKey& eventKey, 124 const LogEvent& event, std::vector<Interval>& intervals, 125 ValueBases& bases) override; 126 127 void pullAndMatchEventsLocked(const int64_t timestampNs) override; 128 129 DumpProtoFields getDumpProtoFields() const override; 130 131 void writePastBucketAggregateToProto(const int aggIndex, const Value& value, 132 ProtoOutputStream* const protoOutput) const override; 133 134 // Internal function to calculate the current used bytes. 135 size_t byteSizeLocked() const override; 136 137 void combineValueFields(pair<LogEvent, vector<int>>& eventValues, const LogEvent& newEvent, 138 const vector<int>& newValueIndices) const; 139 140 const bool mUseAbsoluteValueOnReset; 141 142 const ValueMetric::AggregationType mAggregationType; 143 144 const bool mUseDiff; 145 146 const ValueMetric::ValueDirection mValueDirection; 147 148 const bool mSkipZeroDiffOutput; 149 150 // If true, use a zero value as base to compute the diff. 151 // This is used for new keys which are present in the new data but was not 152 // present in the base data. 153 // The default base will only be used if we have a global base. 154 const bool mUseZeroDefaultBase; 155 156 // For pulled metrics, this is always set to true whenever a pull succeeds. 157 // It is set to false when a pull fails, or upon condition change to false. 158 // This is used to decide if we have the right base data to compute the 159 // diff against. 160 bool mHasGlobalBase; 161 162 const int64_t mMaxPullDelayNs; 163 164 // For anomaly detection. 165 std::unordered_map<MetricDimensionKey, int64_t> mCurrentFullBucket; 166 167 FRIEND_TEST(NumericValueMetricProducerTest, TestAnomalyDetection); 168 FRIEND_TEST(NumericValueMetricProducerTest, TestBaseSetOnConditionChange); 169 FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundariesOnConditionChange); 170 FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryNoCondition); 171 FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryWithCondition); 172 FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryWithCondition2); 173 FRIEND_TEST(NumericValueMetricProducerTest, TestBucketInvalidIfGlobalBaseIsNotSet); 174 FRIEND_TEST(NumericValueMetricProducerTest, TestCalcPreviousBucketEndTime); 175 FRIEND_TEST(NumericValueMetricProducerTest, TestDataIsNotUpdatedWhenNoConditionChanged); 176 FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onBucketBoundary); 177 FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onConditionChanged); 178 FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onDataPulled); 179 FRIEND_TEST(NumericValueMetricProducerTest, TestEventsWithNonSlicedCondition); 180 FRIEND_TEST(NumericValueMetricProducerTest, TestFirstBucket); 181 FRIEND_TEST(NumericValueMetricProducerTest, TestLateOnDataPulledWithDiff); 182 FRIEND_TEST(NumericValueMetricProducerTest, TestLateOnDataPulledWithoutDiff); 183 FRIEND_TEST(NumericValueMetricProducerTest, TestPartialResetOnBucketBoundaries); 184 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryFalse); 185 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryTrue); 186 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withFailure); 187 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withMultipleConditionChanges); 188 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withoutCondition); 189 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsNoCondition); 190 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset); 191 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsTakeZeroOnReset); 192 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsWithFiltering); 193 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledWithAppUpgradeDisabled); 194 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateAvg); 195 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateMax); 196 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateMin); 197 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateSum); 198 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedEventsWithCondition); 199 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedEventsWithoutCondition); 200 FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullDelayExceeded); 201 FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullFailAfterConditionChange); 202 FRIEND_TEST(NumericValueMetricProducerTest, 203 TestResetBaseOnPullFailAfterConditionChange_EndOfBucket); 204 FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullFailBeforeConditionChange); 205 FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullTooLate); 206 FRIEND_TEST(NumericValueMetricProducerTest, TestSkipZeroDiffOutput); 207 FRIEND_TEST(NumericValueMetricProducerTest, TestSkipZeroDiffOutputMultiValue); 208 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedState); 209 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMap); 210 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithPrimaryField_WithDimensions); 211 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithCondition); 212 FRIEND_TEST(NumericValueMetricProducerTest, TestTrimUnusedDimensionKey); 213 FRIEND_TEST(NumericValueMetricProducerTest, TestUseZeroDefaultBase); 214 FRIEND_TEST(NumericValueMetricProducerTest, TestUseZeroDefaultBaseWithPullFailures); 215 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMultipleDimensions); 216 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMissingDataInStateChange); 217 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithDataMissingInConditionChange); 218 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMissingDataThenFlushBucket); 219 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithNoPullOnBucketBoundary); 220 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithConditionFalseMultipleBuckets); 221 FRIEND_TEST(NumericValueMetricProducerTest, 222 TestSlicedStateWithMultipleDimensionsMissingDataInPull); 223 FRIEND_TEST(NumericValueMetricProducerTest, TestUploadThreshold); 224 225 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenOneConditionFailed); 226 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenInitialPullFailed); 227 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenLastPullFailed); 228 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenGuardRailHit); 229 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, 230 TestInvalidBucketWhenDumpReportRequested); 231 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, 232 TestInvalidBucketWhenAccumulateEventWrongBucket); 233 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, 234 TestInvalidBucketWhenMultipleBucketsSkipped); 235 236 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestBucketBoundariesOnPartialBucket); 237 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, 238 TestFullBucketResetWhenLastBucketInvalid); 239 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPartialBucketCreated); 240 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPushedEvents); 241 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPulledValue); 242 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPulledValueWhileConditionFalse); 243 244 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 245 TestAlarmLatePullWhileConditionTrue); 246 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 247 TestAlarmLatePullWithConditionChanged); 248 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 249 TestAlarmLatePullWhileConditionFalse); 250 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 251 TestLatePullOnConditionChangeFalse); 252 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 253 TestLatePullOnConditionChangeTrue); 254 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestAlarmLatePullNoCondition); 255 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 256 TestAlarmLatePullNoConditionWithSkipped); 257 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 258 TestThresholdNotDefinedNoUpload); 259 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestThresholdDefinedZero); 260 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 261 TestThresholdUploadPassWhenEqual); 262 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 263 TestThresholdUploadPassWhenGreater); 264 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestThresholdUploadSkip); 265 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestLateStateChangeSlicedAtoms); 266 267 FRIEND_TEST(NumericValueMetricProducerTest, TestSubsetDimensions); 268 269 FRIEND_TEST(ConfigUpdateTest, TestUpdateValueMetrics); 270 271 friend class NumericValueMetricProducerTestHelper; 272 }; 273 274 } // namespace statsd 275 } // namespace os 276 } // namespace android 277