1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <unordered_map> 20 21 #include <android/util/ProtoOutputStream.h> 22 #include <gtest/gtest_prod.h> 23 #include "../condition/ConditionTracker.h" 24 #include "../external/PullDataReceiver.h" 25 #include "../external/StatsPullerManager.h" 26 #include "../matchers/matcher_util.h" 27 #include "../matchers/EventMatcherWizard.h" 28 #include "MetricProducer.h" 29 #include "src/statsd_config.pb.h" 30 #include "../stats_util.h" 31 32 namespace android { 33 namespace os { 34 namespace statsd { 35 36 struct GaugeAtom { GaugeAtomGaugeAtom37 GaugeAtom(std::vector<FieldValue> fields, int64_t elapsedTimeNs) 38 : mFields(fields), mElapsedTimestampNs(elapsedTimeNs) { 39 } 40 std::vector<FieldValue> mFields; 41 int64_t mElapsedTimestampNs; 42 }; 43 44 struct GaugeBucket { 45 int64_t mBucketStartNs; 46 int64_t mBucketEndNs; 47 std::vector<GaugeAtom> mGaugeAtoms; 48 49 // Maps the field/value pairs of an atom to a list of timestamps used to deduplicate atoms. 50 std::unordered_map<AtomDimensionKey, std::vector<int64_t>> mAggregatedAtoms; 51 }; 52 53 typedef std::unordered_map<MetricDimensionKey, std::vector<GaugeAtom>> 54 DimToGaugeAtomsMap; 55 56 // This gauge metric producer first register the puller to automatically pull the gauge at the 57 // beginning of each bucket. If the condition is met, insert it to the bucket info. Otherwise 58 // proactively pull the gauge when the condition is changed to be true. Therefore, the gauge metric 59 // producer always reports the gauge at the earliest time of the bucket when the condition is met. 60 class GaugeMetricProducer : public MetricProducer, public virtual PullDataReceiver { 61 public: 62 GaugeMetricProducer( 63 const ConfigKey& key, const GaugeMetric& gaugeMetric, int conditionIndex, 64 const vector<ConditionState>& initialConditionCache, 65 const sp<ConditionWizard>& conditionWizard, const uint64_t protoHash, 66 const int whatMatcherIndex, const sp<EventMatcherWizard>& matcherWizard, 67 const int pullTagId, int triggerAtomId, int atomId, const int64_t timeBaseNs, 68 int64_t startTimeNs, const sp<StatsPullerManager>& pullerManager, 69 const wp<ConfigMetadataProvider> configMetadataProvider, 70 const std::unordered_map<int, std::shared_ptr<Activation>>& eventActivationMap = {}, 71 const std::unordered_map<int, std::vector<std::shared_ptr<Activation>>>& 72 eventDeactivationMap = {}, 73 const std::vector<int>& slicedStateAtoms = {}, 74 const std::unordered_map<int, std::unordered_map<int, int64_t>>& stateGroupMap = {}, 75 const size_t dimensionSoftLimit = StatsdStats::kDimensionKeySizeSoftLimit, 76 const size_t dimensionHardLimit = StatsdStats::kDimensionKeySizeHardLimit); 77 78 virtual ~GaugeMetricProducer(); 79 80 // Handles when the pulled data arrives. 81 void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& data, PullResult pullResult, 82 int64_t originalPullTimeNs) override; 83 84 // Determine if metric needs to pull isPullNeeded()85 bool isPullNeeded() const override { 86 std::lock_guard<std::mutex> lock(mMutex); 87 return mIsActive && (mCondition == ConditionState::kTrue) && 88 shouldKeepRandomSample(mPullProbability); 89 }; 90 91 // GaugeMetric needs to immediately trigger another pull when we create the partial bucket. notifyAppUpgradeInternalLocked(int64_t eventTimeNs)92 void notifyAppUpgradeInternalLocked(int64_t eventTimeNs) override { 93 flushLocked(eventTimeNs); 94 if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE && mIsActive) { 95 pullAndMatchEventsLocked(eventTimeNs); 96 } 97 }; 98 99 // GaugeMetric needs to immediately trigger another pull when we create the partial bucket. onStatsdInitCompleted(int64_t eventTimeNs)100 void onStatsdInitCompleted(int64_t eventTimeNs) override { 101 ATRACE_CALL(); 102 std::lock_guard<std::mutex> lock(mMutex); 103 flushLocked(eventTimeNs); 104 if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE && mIsActive) { 105 pullAndMatchEventsLocked(eventTimeNs); 106 } 107 }; 108 getMetricType()109 MetricType getMetricType() const override { 110 return METRIC_TYPE_GAUGE; 111 } 112 113 void onStateChanged(const int64_t eventTimeNs, const int32_t atomId, 114 const HashableDimensionKey& primaryKey, const FieldValue& oldState, 115 const FieldValue& newState) override; 116 117 protected: 118 void onMatchedLogEventInternalLocked( 119 const size_t matcherIndex, const MetricDimensionKey& eventKey, 120 const ConditionKey& conditionKey, bool condition, const LogEvent& event, 121 const std::map<int, HashableDimensionKey>& statePrimaryKeys) override; 122 123 private: 124 void onDumpReportLocked(const int64_t dumpTimeNs, const bool include_current_partial_bucket, 125 const bool erase_data, const DumpLatency dumpLatency, 126 std::set<string>* str_set, std::set<int32_t>& usedUids, 127 android::util::ProtoOutputStream* protoOutput) override; 128 void clearPastBucketsLocked(const int64_t dumpTimeNs) override; 129 130 // Internal interface to handle condition change. 131 void onConditionChangedLocked(const bool conditionMet, int64_t eventTime) override; 132 133 // Internal interface to handle active state change. 134 void onActiveStateChangedLocked(const int64_t eventTimeNs, const bool isActive) override; 135 136 // Internal interface to handle sliced condition change. 137 void onSlicedConditionMayChangeLocked(bool overallCondition, int64_t eventTime) override; 138 139 // Internal function to calculate the current used bytes. 140 size_t byteSizeLocked() const override; 141 142 void dumpStatesLocked(int out, bool verbose) const override; 143 144 void dropDataLocked(const int64_t dropTimeNs) override; 145 146 // Util function to flush the old packet. 147 void flushIfNeededLocked(int64_t eventTime) override; 148 149 void flushCurrentBucketLocked(int64_t eventTimeNs, int64_t nextBucketStartTimeNs) override; 150 151 void prepareFirstBucketLocked() override; 152 153 // Only call if mCondition == ConditionState::kTrue && metric is active. 154 void pullAndMatchEventsLocked(const int64_t timestampNs); 155 156 size_t computeGaugeBucketSizeLocked( 157 const bool isFullBucket, const MetricDimensionKey& dimKey, const bool isFirstBucket, 158 const std::unordered_map<AtomDimensionKey, std::vector<int64_t>>& aggregatedAtoms) 159 const; 160 161 optional<InvalidConfigReason> onConfigUpdatedLocked( 162 const StatsdConfig& config, int configIndex, int metricIndex, 163 const std::vector<sp<AtomMatchingTracker>>& allAtomMatchingTrackers, 164 const std::unordered_map<int64_t, int>& oldAtomMatchingTrackerMap, 165 const std::unordered_map<int64_t, int>& newAtomMatchingTrackerMap, 166 const sp<EventMatcherWizard>& matcherWizard, 167 const std::vector<sp<ConditionTracker>>& allConditionTrackers, 168 const std::unordered_map<int64_t, int>& conditionTrackerMap, 169 const sp<ConditionWizard>& wizard, 170 const std::unordered_map<int64_t, int>& metricToActivationMap, 171 std::unordered_map<int, std::vector<int>>& trackerToMetricMap, 172 std::unordered_map<int, std::vector<int>>& conditionToMetricMap, 173 std::unordered_map<int, std::vector<int>>& activationAtomTrackerToMetricMap, 174 std::unordered_map<int, std::vector<int>>& deactivationAtomTrackerToMetricMap, 175 std::vector<int>& metricsWithActivation) override; 176 isRandomNSamples()177 inline bool isRandomNSamples() const { 178 return (mTriggerAtomId == -1 && mSamplingType == GaugeMetric::FIRST_N_SAMPLES) || 179 mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE; 180 } 181 182 DataCorruptionSeverity determineCorruptionSeverity(int32_t atomId, DataCorruptedReason reason, 183 LostAtomType atomType) const override; 184 185 int mWhatMatcherIndex; 186 187 sp<EventMatcherWizard> mEventMatcherWizard; 188 189 sp<StatsPullerManager> mPullerManager; 190 191 // tagId for pulled data. -1 if this is not pulled 192 const int mPullTagId; 193 194 // tagId for atoms that trigger the pulling, if any 195 const int mTriggerAtomId; 196 197 // tagId for output atom 198 const int mAtomId; 199 200 // if this is pulled metric 201 const bool mIsPulled; 202 203 // Save the past buckets and we can clear when the StatsLogReport is dumped. 204 std::unordered_map<MetricDimensionKey, std::vector<GaugeBucket>> mPastBuckets; 205 206 // The current partial bucket. 207 std::shared_ptr<DimToGaugeAtomsMap> mCurrentSlicedBucket; 208 209 // The current full bucket for anomaly detection. This is updated to the latest value seen for 210 // this slice (ie, for partial buckets, we use the last partial bucket in this full bucket). 211 std::shared_ptr<DimToValMap> mCurrentSlicedBucketForAnomaly; 212 213 const int64_t mMinBucketSizeNs; 214 215 // Translate Atom based bucket to single numeric value bucket for anomaly and updates the map 216 // for each slice with the latest value. 217 void updateCurrentSlicedBucketForAnomaly(); 218 219 // Allowlist/denylist of fields to report. Empty means all are reported. 220 // If mOmitFields == true, this is a denylist, otherwise it's an allowlist. 221 const std::vector<Matcher> mFieldMatchers; 222 const bool mOmitFields; 223 224 GaugeMetric::SamplingType mSamplingType; 225 226 const int64_t mMaxPullDelayNs; 227 228 // apply an allowlist on the original input 229 std::vector<FieldValue> getGaugeFields(const LogEvent& event); 230 231 // Util function to check whether the specified dimension hits the guardrail. 232 bool hitGuardRailLocked(const MetricDimensionKey& newKey); 233 234 static const size_t kBucketSize = sizeof(GaugeBucket{}); 235 236 const size_t mDimensionSoftLimit; 237 238 const size_t mDimensionHardLimit; 239 240 const size_t mGaugeAtomsPerDimensionLimit; 241 242 // Tracks if the dimension guardrail has been hit in the current report. 243 bool mDimensionGuardrailHit; 244 245 const int mSamplingPercentage; 246 247 const int mPullProbability; 248 249 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsWithCondition); 250 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsWithSlicedCondition); 251 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsNoCondition); 252 FRIEND_TEST(GaugeMetricProducerTest, TestPulledWithAppUpgradeDisabled); 253 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsAnomalyDetection); 254 FRIEND_TEST(GaugeMetricProducerTest, TestFirstBucket); 255 FRIEND_TEST(GaugeMetricProducerTest, TestPullOnTrigger); 256 FRIEND_TEST(GaugeMetricProducerTest, TestPullNWithoutTrigger); 257 FRIEND_TEST(GaugeMetricProducerTest, TestRemoveDimensionInOutput); 258 FRIEND_TEST(GaugeMetricProducerTest, TestPullDimensionalSampling); 259 260 FRIEND_TEST(GaugeMetricProducerTest_PartialBucket, TestPushedEvents); 261 FRIEND_TEST(GaugeMetricProducerTest_PartialBucket, TestPulled); 262 263 FRIEND_TEST(ConfigUpdateTest, TestUpdateGaugeMetrics); 264 265 FRIEND_TEST(MetricsManagerUtilDimLimitTest, TestDimLimit); 266 267 FRIEND_TEST(ConfigUpdateDimLimitTest, TestDimLimit); 268 }; 269 270 } // namespace statsd 271 } // namespace os 272 } // namespace android 273