1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <gtest/gtest_prod.h> 20 21 #include <optional> 22 23 #include "ValueMetricProducer.h" 24 25 namespace android { 26 namespace os { 27 namespace statsd { 28 29 // TODO(b/185796344): don't use Value from FieldValue. 30 using ValueBases = std::vector<std::optional<Value>>; 31 class NumericValueMetricProducer : public ValueMetricProducer<Value, ValueBases> { 32 public: 33 NumericValueMetricProducer(const ConfigKey& key, const ValueMetric& valueMetric, 34 const uint64_t protoHash, const PullOptions& pullOptions, 35 const BucketOptions& bucketOptions, const WhatOptions& whatOptions, 36 const ConditionOptions& conditionOptions, 37 const StateOptions& stateOptions, 38 const ActivationOptions& activationOptions, 39 const GuardrailOptions& guardrailOptions); 40 41 // Process data pulled on bucket boundary. 42 void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& allData, PullResult pullResult, 43 int64_t originalPullTimeNs) override; 44 45 // Determine if metric needs to pull isPullNeeded()46 bool isPullNeeded() const override { 47 std::lock_guard<std::mutex> lock(mMutex); 48 return mIsActive && (mCondition == ConditionState::kTrue); 49 } 50 getMetricType()51 inline MetricType getMetricType() const override { 52 return METRIC_TYPE_VALUE; 53 } 54 55 protected: 56 private: 57 void prepareFirstBucketLocked() override; 58 getConditionIdForMetric(const StatsdConfig & config,const int configIndex)59 inline optional<int64_t> getConditionIdForMetric(const StatsdConfig& config, 60 const int configIndex) const override { 61 const ValueMetric& metric = config.value_metric(configIndex); 62 return metric.has_condition() ? make_optional(metric.condition()) : nullopt; 63 } 64 getWhatAtomMatcherIdForMetric(const StatsdConfig & config,const int configIndex)65 inline int64_t getWhatAtomMatcherIdForMetric(const StatsdConfig& config, 66 const int configIndex) const override { 67 return config.value_metric(configIndex).what(); 68 } 69 getConditionLinksForMetric(const StatsdConfig & config,const int configIndex)70 inline ConditionLinks getConditionLinksForMetric(const StatsdConfig& config, 71 const int configIndex) const override { 72 return config.value_metric(configIndex).links(); 73 } 74 75 void onActiveStateChangedInternalLocked(const int64_t eventTimeNs, 76 const bool isActive) override; 77 78 // Only called when mIsActive and the event is NOT too late. 79 void onConditionChangedInternalLocked(const ConditionState oldCondition, 80 const ConditionState newCondition, 81 const int64_t eventTimeNs) override; 82 aggregatedValueToString(const Value & value)83 inline std::string aggregatedValueToString(const Value& value) const override { 84 return value.toString(); 85 } 86 87 // Mark the data as invalid. 88 void invalidateCurrentBucket(const int64_t dropTimeNs, const BucketDropReason reason) override; 89 90 // Reset diff base and mHasGlobalBase 91 void resetBase(); 92 93 // Calculate previous bucket end time based on current time. 94 int64_t calcPreviousBucketEndTime(const int64_t currentTimeNs); 95 multipleBucketsSkipped(const int64_t numBucketsForward)96 inline bool multipleBucketsSkipped(const int64_t numBucketsForward) const override { 97 return numBucketsForward > 1 && (isPulled() || mUseDiff); 98 } 99 100 // Process events retrieved from a pull. 101 void accumulateEvents(const std::vector<std::shared_ptr<LogEvent>>& allData, 102 int64_t originalPullTimeNs, int64_t eventElapsedTimeNs); 103 104 void closeCurrentBucket(const int64_t eventTimeNs, 105 const int64_t nextBucketStartTimeNs) override; 106 107 PastBucket<Value> buildPartialBucket(int64_t bucketEndTime, 108 std::vector<Interval>& intervals) override; 109 110 bool valuePassesThreshold(const Interval& interval) const; 111 112 Value getFinalValue(const Interval& interval) const; 113 114 void initNextSlicedBucket(int64_t nextBucketStartTimeNs) override; 115 116 void appendToFullBucket(const bool isFullBucketReached); 117 118 bool hitFullBucketGuardRailLocked(const MetricDimensionKey& newKey); 119 canSkipLogEventLocked(const MetricDimensionKey & eventKey,const bool condition,const int64_t eventTimeNs,const map<int,HashableDimensionKey> & statePrimaryKeys)120 inline bool canSkipLogEventLocked( 121 const MetricDimensionKey& eventKey, const bool condition, const int64_t eventTimeNs, 122 const map<int, HashableDimensionKey>& statePrimaryKeys) const override { 123 // For pushed metrics, can only skip if condition is false. 124 // For pulled metrics, can only skip if metric is not diffed and condition is false or 125 // unknown. 126 return (!isPulled() && !condition) || 127 (isPulled() && !mUseDiff && mCondition != ConditionState::kTrue); 128 } 129 130 bool aggregateFields(const int64_t eventTimeNs, const MetricDimensionKey& eventKey, 131 const LogEvent& event, std::vector<Interval>& intervals, 132 ValueBases& bases) override; 133 134 void pullAndMatchEventsLocked(const int64_t timestampNs) override; 135 136 DumpProtoFields getDumpProtoFields() const override; 137 138 void writePastBucketAggregateToProto(const int aggIndex, const Value& value, 139 const int sampleSize, 140 ProtoOutputStream* const protoOutput) const override; 141 142 // Internal function to calculate the current used bytes. 143 size_t byteSizeLocked() const override; 144 145 void combineValueFields(pair<LogEvent, vector<int>>& eventValues, const LogEvent& newEvent, 146 const vector<int>& newValueIndices) const; 147 148 const bool mUseAbsoluteValueOnReset; 149 150 const ValueMetric::AggregationType mAggregationType; 151 152 const bool mIncludeSampleSize; 153 154 const bool mUseDiff; 155 156 const ValueMetric::ValueDirection mValueDirection; 157 158 const bool mSkipZeroDiffOutput; 159 160 // If true, use a zero value as base to compute the diff. 161 // This is used for new keys which are present in the new data but was not 162 // present in the base data. 163 // The default base will only be used if we have a global base. 164 const bool mUseZeroDefaultBase; 165 166 // For pulled metrics, this is always set to true whenever a pull succeeds. 167 // It is set to false when a pull fails, or upon condition change to false. 168 // This is used to decide if we have the right base data to compute the 169 // diff against. 170 bool mHasGlobalBase; 171 172 const int64_t mMaxPullDelayNs; 173 174 // For anomaly detection. 175 std::unordered_map<MetricDimensionKey, int64_t> mCurrentFullBucket; 176 177 FRIEND_TEST(NumericValueMetricProducerTest, TestAnomalyDetection); 178 FRIEND_TEST(NumericValueMetricProducerTest, TestBaseSetOnConditionChange); 179 FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundariesOnConditionChange); 180 FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryNoCondition); 181 FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryWithCondition); 182 FRIEND_TEST(NumericValueMetricProducerTest, TestBucketBoundaryWithCondition2); 183 FRIEND_TEST(NumericValueMetricProducerTest, TestBucketInvalidIfGlobalBaseIsNotSet); 184 FRIEND_TEST(NumericValueMetricProducerTest, TestCalcPreviousBucketEndTime); 185 FRIEND_TEST(NumericValueMetricProducerTest, TestDataIsNotUpdatedWhenNoConditionChanged); 186 FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onBucketBoundary); 187 FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onConditionChanged); 188 FRIEND_TEST(NumericValueMetricProducerTest, TestEmptyDataResetsBase_onDataPulled); 189 FRIEND_TEST(NumericValueMetricProducerTest, TestEventsWithNonSlicedCondition); 190 FRIEND_TEST(NumericValueMetricProducerTest, TestFirstBucket); 191 FRIEND_TEST(NumericValueMetricProducerTest, TestLateOnDataPulledWithDiff); 192 FRIEND_TEST(NumericValueMetricProducerTest, TestLateOnDataPulledWithoutDiff); 193 FRIEND_TEST(NumericValueMetricProducerTest, TestPartialResetOnBucketBoundaries); 194 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryFalse); 195 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryTrue); 196 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withFailure); 197 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withMultipleConditionChanges); 198 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledData_noDiff_withoutCondition); 199 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsNoCondition); 200 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset); 201 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsTakeZeroOnReset); 202 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledEventsWithFiltering); 203 FRIEND_TEST(NumericValueMetricProducerTest, TestPulledWithAppUpgradeDisabled); 204 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateAvg); 205 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateMax); 206 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateMin); 207 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedAggregateSum); 208 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedEventsWithCondition); 209 FRIEND_TEST(NumericValueMetricProducerTest, TestPushedEventsWithoutCondition); 210 FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullDelayExceeded); 211 FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullFailAfterConditionChange); 212 FRIEND_TEST(NumericValueMetricProducerTest, 213 TestResetBaseOnPullFailAfterConditionChange_EndOfBucket); 214 FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullFailBeforeConditionChange); 215 FRIEND_TEST(NumericValueMetricProducerTest, TestResetBaseOnPullTooLate); 216 FRIEND_TEST(NumericValueMetricProducerTest, TestSampleSize); 217 FRIEND_TEST(NumericValueMetricProducerTest, TestSkipZeroDiffOutput); 218 FRIEND_TEST(NumericValueMetricProducerTest, TestSkipZeroDiffOutputMultiValue); 219 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedState); 220 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMap); 221 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithPrimaryField_WithDimensions); 222 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithCondition); 223 FRIEND_TEST(NumericValueMetricProducerTest, TestTrimUnusedDimensionKey); 224 FRIEND_TEST(NumericValueMetricProducerTest, TestUseZeroDefaultBase); 225 FRIEND_TEST(NumericValueMetricProducerTest, TestUseZeroDefaultBaseWithPullFailures); 226 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMultipleDimensions); 227 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMissingDataInStateChange); 228 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithDataMissingInConditionChange); 229 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithMissingDataThenFlushBucket); 230 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithNoPullOnBucketBoundary); 231 FRIEND_TEST(NumericValueMetricProducerTest, TestSlicedStateWithConditionFalseMultipleBuckets); 232 FRIEND_TEST(NumericValueMetricProducerTest, 233 TestSlicedStateWithMultipleDimensionsMissingDataInPull); 234 FRIEND_TEST(NumericValueMetricProducerTest, TestUploadThreshold); 235 236 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenOneConditionFailed); 237 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenInitialPullFailed); 238 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenLastPullFailed); 239 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, TestInvalidBucketWhenGuardRailHit); 240 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, 241 TestInvalidBucketWhenDumpReportRequested); 242 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, 243 TestInvalidBucketWhenAccumulateEventWrongBucket); 244 FRIEND_TEST(NumericValueMetricProducerTest_BucketDrop, 245 TestInvalidBucketWhenMultipleBucketsSkipped); 246 247 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestBucketBoundariesOnPartialBucket); 248 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, 249 TestFullBucketResetWhenLastBucketInvalid); 250 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPartialBucketCreated); 251 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPushedEvents); 252 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPulledValue); 253 FRIEND_TEST(NumericValueMetricProducerTest_PartialBucket, TestPulledValueWhileConditionFalse); 254 255 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 256 TestAlarmLatePullWhileConditionTrue); 257 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 258 TestAlarmLatePullWithConditionChanged); 259 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 260 TestAlarmLatePullWhileConditionFalse); 261 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 262 TestLatePullOnConditionChangeFalse); 263 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 264 TestLatePullOnConditionChangeTrue); 265 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestAlarmLatePullNoCondition); 266 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 267 TestAlarmLatePullNoConditionWithSkipped); 268 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 269 TestThresholdNotDefinedNoUpload); 270 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestThresholdDefinedZero); 271 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 272 TestThresholdUploadPassWhenEqual); 273 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, 274 TestThresholdUploadPassWhenGreater); 275 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestThresholdUploadSkip); 276 FRIEND_TEST(NumericValueMetricProducerTest_ConditionCorrection, TestLateStateChangeSlicedAtoms); 277 278 FRIEND_TEST(NumericValueMetricProducerTest, TestSubsetDimensions); 279 280 FRIEND_TEST(ConfigUpdateTest, TestUpdateValueMetrics); 281 282 friend class NumericValueMetricProducerTestHelper; 283 }; 284 285 } // namespace statsd 286 } // namespace os 287 } // namespace android 288