1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <unordered_map> 20 21 #include <android/util/ProtoOutputStream.h> 22 #include <gtest/gtest_prod.h> 23 #include "../condition/ConditionTracker.h" 24 #include "../external/PullDataReceiver.h" 25 #include "../external/StatsPullerManager.h" 26 #include "../matchers/matcher_util.h" 27 #include "../matchers/EventMatcherWizard.h" 28 #include "MetricProducer.h" 29 #include "src/statsd_config.pb.h" 30 #include "../stats_util.h" 31 32 namespace android { 33 namespace os { 34 namespace statsd { 35 36 struct GaugeAtom { GaugeAtomGaugeAtom37 GaugeAtom(std::shared_ptr<vector<FieldValue>> fields, int64_t elapsedTimeNs) 38 : mFields(fields), mElapsedTimestampNs(elapsedTimeNs) { 39 } 40 std::shared_ptr<vector<FieldValue>> mFields; 41 int64_t mElapsedTimestampNs; 42 }; 43 44 struct GaugeBucket { 45 int64_t mBucketStartNs; 46 int64_t mBucketEndNs; 47 std::vector<GaugeAtom> mGaugeAtoms; 48 49 // Maps the field/value pairs of an atom to a list of timestamps used to deduplicate atoms. 50 std::unordered_map<AtomDimensionKey, std::vector<int64_t>> mAggregatedAtoms; 51 }; 52 53 typedef std::unordered_map<MetricDimensionKey, std::vector<GaugeAtom>> 54 DimToGaugeAtomsMap; 55 56 // This gauge metric producer first register the puller to automatically pull the gauge at the 57 // beginning of each bucket. If the condition is met, insert it to the bucket info. Otherwise 58 // proactively pull the gauge when the condition is changed to be true. Therefore, the gauge metric 59 // producer always reports the gauge at the earliest time of the bucket when the condition is met. 60 class GaugeMetricProducer : public MetricProducer, public virtual PullDataReceiver { 61 public: 62 GaugeMetricProducer( 63 const ConfigKey& key, const GaugeMetric& gaugeMetric, const int conditionIndex, 64 const vector<ConditionState>& initialConditionCache, 65 const sp<ConditionWizard>& conditionWizard, const uint64_t protoHash, 66 const int whatMatcherIndex, const sp<EventMatcherWizard>& matcherWizard, 67 const int pullTagId, const int triggerAtomId, const int atomId, 68 const int64_t timeBaseNs, const int64_t startTimeNs, 69 const sp<StatsPullerManager>& pullerManager, 70 const std::unordered_map<int, std::shared_ptr<Activation>>& eventActivationMap = {}, 71 const std::unordered_map<int, std::vector<std::shared_ptr<Activation>>>& 72 eventDeactivationMap = {}, 73 const size_t dimensionSoftLimit = StatsdStats::kDimensionKeySizeSoftLimit, 74 const size_t dimensionHardLimit = StatsdStats::kDimensionKeySizeHardLimit); 75 76 virtual ~GaugeMetricProducer(); 77 78 // Handles when the pulled data arrives. 79 void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& data, PullResult pullResult, 80 int64_t originalPullTimeNs) override; 81 82 // Determine if metric needs to pull isPullNeeded()83 bool isPullNeeded() const override { 84 std::lock_guard<std::mutex> lock(mMutex); 85 return mIsActive && (mCondition == ConditionState::kTrue); 86 }; 87 88 // GaugeMetric needs to immediately trigger another pull when we create the partial bucket. notifyAppUpgradeInternalLocked(const int64_t eventTimeNs)89 void notifyAppUpgradeInternalLocked(const int64_t eventTimeNs) override { 90 flushLocked(eventTimeNs); 91 if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE && mIsActive) { 92 pullAndMatchEventsLocked(eventTimeNs); 93 } 94 }; 95 96 // GaugeMetric needs to immediately trigger another pull when we create the partial bucket. onStatsdInitCompleted(const int64_t & eventTimeNs)97 void onStatsdInitCompleted(const int64_t& eventTimeNs) override { 98 std::lock_guard<std::mutex> lock(mMutex); 99 100 flushLocked(eventTimeNs); 101 if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE && mIsActive) { 102 pullAndMatchEventsLocked(eventTimeNs); 103 } 104 }; 105 getMetricType()106 MetricType getMetricType() const override { 107 return METRIC_TYPE_GAUGE; 108 } 109 110 protected: 111 void onMatchedLogEventInternalLocked( 112 const size_t matcherIndex, const MetricDimensionKey& eventKey, 113 const ConditionKey& conditionKey, bool condition, const LogEvent& event, 114 const std::map<int, HashableDimensionKey>& statePrimaryKeys) override; 115 116 private: 117 void onDumpReportLocked(const int64_t dumpTimeNs, 118 const bool include_current_partial_bucket, 119 const bool erase_data, 120 const DumpLatency dumpLatency, 121 std::set<string> *str_set, 122 android::util::ProtoOutputStream* protoOutput) override; 123 void clearPastBucketsLocked(const int64_t dumpTimeNs) override; 124 125 // Internal interface to handle condition change. 126 void onConditionChangedLocked(const bool conditionMet, const int64_t eventTime) override; 127 128 // Internal interface to handle active state change. 129 void onActiveStateChangedLocked(const int64_t eventTimeNs, const bool isActive) override; 130 131 // Internal interface to handle sliced condition change. 132 void onSlicedConditionMayChangeLocked(bool overallCondition, const int64_t eventTime) override; 133 134 // Internal function to calculate the current used bytes. 135 size_t byteSizeLocked() const override; 136 137 void dumpStatesLocked(FILE* out, bool verbose) const override; 138 139 void dropDataLocked(const int64_t dropTimeNs) override; 140 141 // Util function to flush the old packet. 142 void flushIfNeededLocked(const int64_t& eventTime) override; 143 144 void flushCurrentBucketLocked(const int64_t& eventTimeNs, 145 const int64_t& nextBucketStartTimeNs) override; 146 147 void prepareFirstBucketLocked() override; 148 149 // Only call if mCondition == ConditionState::kTrue && metric is active. 150 void pullAndMatchEventsLocked(const int64_t timestampNs); 151 152 optional<InvalidConfigReason> onConfigUpdatedLocked( 153 const StatsdConfig& config, const int configIndex, const int metricIndex, 154 const std::vector<sp<AtomMatchingTracker>>& allAtomMatchingTrackers, 155 const std::unordered_map<int64_t, int>& oldAtomMatchingTrackerMap, 156 const std::unordered_map<int64_t, int>& newAtomMatchingTrackerMap, 157 const sp<EventMatcherWizard>& matcherWizard, 158 const std::vector<sp<ConditionTracker>>& allConditionTrackers, 159 const std::unordered_map<int64_t, int>& conditionTrackerMap, 160 const sp<ConditionWizard>& wizard, 161 const std::unordered_map<int64_t, int>& metricToActivationMap, 162 std::unordered_map<int, std::vector<int>>& trackerToMetricMap, 163 std::unordered_map<int, std::vector<int>>& conditionToMetricMap, 164 std::unordered_map<int, std::vector<int>>& activationAtomTrackerToMetricMap, 165 std::unordered_map<int, std::vector<int>>& deactivationAtomTrackerToMetricMap, 166 std::vector<int>& metricsWithActivation) override; 167 isRandomNSamples()168 inline bool isRandomNSamples() const { 169 return (mTriggerAtomId == -1 && mSamplingType == GaugeMetric::FIRST_N_SAMPLES) || 170 mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE; 171 } 172 173 int mWhatMatcherIndex; 174 175 sp<EventMatcherWizard> mEventMatcherWizard; 176 177 sp<StatsPullerManager> mPullerManager; 178 // tagId for pulled data. -1 if this is not pulled 179 const int mPullTagId; 180 181 // tagId for atoms that trigger the pulling, if any 182 const int mTriggerAtomId; 183 184 // tagId for output atom 185 const int mAtomId; 186 187 // if this is pulled metric 188 const bool mIsPulled; 189 190 // Save the past buckets and we can clear when the StatsLogReport is dumped. 191 std::unordered_map<MetricDimensionKey, std::vector<GaugeBucket>> mPastBuckets; 192 193 // The current partial bucket. 194 std::shared_ptr<DimToGaugeAtomsMap> mCurrentSlicedBucket; 195 196 // The current full bucket for anomaly detection. This is updated to the latest value seen for 197 // this slice (ie, for partial buckets, we use the last partial bucket in this full bucket). 198 std::shared_ptr<DimToValMap> mCurrentSlicedBucketForAnomaly; 199 200 const int64_t mMinBucketSizeNs; 201 202 // Translate Atom based bucket to single numeric value bucket for anomaly and updates the map 203 // for each slice with the latest value. 204 void updateCurrentSlicedBucketForAnomaly(); 205 206 // Allowlist of fields to report. Empty means all are reported. 207 std::vector<Matcher> mFieldMatchers; 208 209 GaugeMetric::SamplingType mSamplingType; 210 211 const int64_t mMaxPullDelayNs; 212 213 // apply an allowlist on the original input 214 std::shared_ptr<vector<FieldValue>> getGaugeFields(const LogEvent& event); 215 216 // Util function to check whether the specified dimension hits the guardrail. 217 bool hitGuardRailLocked(const MetricDimensionKey& newKey); 218 219 static const size_t kBucketSize = sizeof(GaugeBucket{}); 220 221 const size_t mDimensionSoftLimit; 222 223 const size_t mDimensionHardLimit; 224 225 const size_t mGaugeAtomsPerDimensionLimit; 226 227 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsWithCondition); 228 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsWithSlicedCondition); 229 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsNoCondition); 230 FRIEND_TEST(GaugeMetricProducerTest, TestPulledWithAppUpgradeDisabled); 231 FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsAnomalyDetection); 232 FRIEND_TEST(GaugeMetricProducerTest, TestFirstBucket); 233 FRIEND_TEST(GaugeMetricProducerTest, TestPullOnTrigger); 234 FRIEND_TEST(GaugeMetricProducerTest, TestPullNWithoutTrigger); 235 FRIEND_TEST(GaugeMetricProducerTest, TestRemoveDimensionInOutput); 236 FRIEND_TEST(GaugeMetricProducerTest, TestPullDimensionalSampling); 237 238 FRIEND_TEST(GaugeMetricProducerTest_PartialBucket, TestPushedEvents); 239 FRIEND_TEST(GaugeMetricProducerTest_PartialBucket, TestPulled); 240 241 FRIEND_TEST(ConfigUpdateTest, TestUpdateGaugeMetrics); 242 }; 243 244 } // namespace statsd 245 } // namespace os 246 } // namespace android 247