1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include "anomaly/AlarmMonitor.h" 20 #include "anomaly/AlarmTracker.h" 21 #include "anomaly/AnomalyTracker.h" 22 #include "condition/ConditionTracker.h" 23 #include "config/ConfigKey.h" 24 #include "external/StatsPullerManager.h" 25 #include "src/statsd_config.pb.h" 26 #include "src/statsd_metadata.pb.h" 27 #include "logd/LogEvent.h" 28 #include "matchers/AtomMatchingTracker.h" 29 #include "metrics/MetricProducer.h" 30 #include "packages/UidMap.h" 31 32 #include <unordered_map> 33 34 namespace android { 35 namespace os { 36 namespace statsd { 37 38 // A MetricsManager is responsible for managing metrics from one single config source. 39 class MetricsManager : public virtual RefBase, public virtual PullUidProvider { 40 public: 41 MetricsManager(const ConfigKey& configKey, const StatsdConfig& config, const int64_t timeBaseNs, 42 const int64_t currentTimeNs, const sp<UidMap>& uidMap, 43 const sp<StatsPullerManager>& pullerManager, 44 const sp<AlarmMonitor>& anomalyAlarmMonitor, 45 const sp<AlarmMonitor>& periodicAlarmMonitor); 46 47 virtual ~MetricsManager(); 48 49 bool updateConfig(const StatsdConfig& config, const int64_t timeBaseNs, 50 const int64_t currentTimeNs, const sp<AlarmMonitor>& anomalyAlarmMonitor, 51 const sp<AlarmMonitor>& periodicAlarmMonitor); 52 53 // Return whether the configuration is valid. 54 bool isConfigValid() const; 55 56 bool checkLogCredentials(const LogEvent& event); 57 58 bool eventSanityCheck(const LogEvent& event); 59 60 void onLogEvent(const LogEvent& event); 61 62 void onAnomalyAlarmFired( 63 const int64_t& timestampNs, 64 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>>& alarmSet); 65 66 void onPeriodicAlarmFired( 67 const int64_t& timestampNs, 68 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>>& alarmSet); 69 70 void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid, 71 const int64_t version); 72 73 void notifyAppRemoved(const int64_t& eventTimeNs, const string& apk, const int uid); 74 75 void onUidMapReceived(const int64_t& eventTimeNs); 76 77 void onStatsdInitCompleted(const int64_t& elapsedTimeNs); 78 79 void init(); 80 81 vector<int32_t> getPullAtomUids(int32_t atomId) override; 82 shouldWriteToDisk()83 bool shouldWriteToDisk() const { 84 return mNoReportMetricIds.size() != mAllMetricProducers.size(); 85 } 86 shouldPersistLocalHistory()87 bool shouldPersistLocalHistory() const { 88 return mShouldPersistHistory; 89 } 90 91 void dumpStates(FILE* out, bool verbose); 92 isInTtl(const int64_t timestampNs)93 inline bool isInTtl(const int64_t timestampNs) const { 94 return mTtlNs <= 0 || timestampNs < mTtlEndNs; 95 }; 96 hashStringInReport()97 inline bool hashStringInReport() const { 98 return mHashStringsInReport; 99 }; 100 versionStringsInReport()101 inline bool versionStringsInReport() const { 102 return mVersionStringsInReport; 103 }; 104 installerInReport()105 inline bool installerInReport() const { 106 return mInstallerInReport; 107 }; 108 packageCertificateHashSizeBytes()109 inline uint8_t packageCertificateHashSizeBytes() const { 110 return mPackageCertificateHashSizeBytes; 111 } 112 refreshTtl(const int64_t currentTimestampNs)113 void refreshTtl(const int64_t currentTimestampNs) { 114 if (mTtlNs > 0) { 115 mTtlEndNs = currentTimestampNs + mTtlNs; 116 } 117 }; 118 119 // Returns the elapsed realtime when this metric manager last reported metrics. If this config 120 // has not yet dumped any reports, this is the time the metricsmanager was initialized. getLastReportTimeNs()121 inline int64_t getLastReportTimeNs() const { 122 return mLastReportTimeNs; 123 }; 124 getLastReportWallClockNs()125 inline int64_t getLastReportWallClockNs() const { 126 return mLastReportWallClockNs; 127 }; 128 getNumMetrics()129 inline size_t getNumMetrics() const { 130 return mAllMetricProducers.size(); 131 } 132 133 virtual void dropData(const int64_t dropTimeNs); 134 135 virtual void onDumpReport(const int64_t dumpTimeNs, const int64_t wallClockNs, 136 const bool include_current_partial_bucket, const bool erase_data, 137 const DumpLatency dumpLatency, std::set<string>* str_set, 138 android::util::ProtoOutputStream* protoOutput); 139 140 // Computes the total byte size of all metrics managed by a single config source. 141 // Does not change the state. 142 virtual size_t byteSize(); 143 144 // Returns whether or not this config is active. 145 // The config is active if any metric in the config is active. isActive()146 inline bool isActive() const { 147 return mIsActive; 148 } 149 150 void loadActiveConfig(const ActiveConfig& config, int64_t currentTimeNs); 151 152 void writeActiveConfigToProtoOutputStream( 153 int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto); 154 155 // Returns true if at least one piece of metadata is written. 156 bool writeMetadataToProto(int64_t currentWallClockTimeNs, 157 int64_t systemElapsedTimeNs, 158 metadata::StatsMetadata* statsMetadata); 159 160 void loadMetadata(const metadata::StatsMetadata& metadata, 161 int64_t currentWallClockTimeNs, 162 int64_t systemElapsedTimeNs); 163 private: 164 // For test only. getTtlEndNs()165 inline int64_t getTtlEndNs() const { return mTtlEndNs; } 166 167 const ConfigKey mConfigKey; 168 169 sp<UidMap> mUidMap; 170 171 bool mConfigValid = false; 172 173 bool mHashStringsInReport = false; 174 bool mVersionStringsInReport = false; 175 bool mInstallerInReport = false; 176 uint8_t mPackageCertificateHashSizeBytes; 177 178 int64_t mTtlNs; 179 int64_t mTtlEndNs; 180 181 int64_t mLastReportTimeNs; 182 int64_t mLastReportWallClockNs; 183 184 sp<StatsPullerManager> mPullerManager; 185 186 // The uid log sources from StatsdConfig. 187 std::vector<int32_t> mAllowedUid; 188 189 // The pkg log sources from StatsdConfig. 190 std::vector<std::string> mAllowedPkg; 191 192 // The combined uid sources (after translating pkg name to uid). 193 // Logs from uids that are not in the list will be ignored to avoid spamming. 194 std::set<int32_t> mAllowedLogSources; 195 196 // To guard access to mAllowedLogSources 197 mutable std::mutex mAllowedLogSourcesMutex; 198 199 std::set<int32_t> mWhitelistedAtomIds; 200 201 // We can pull any atom from these uids. 202 std::set<int32_t> mDefaultPullUids; 203 204 // Uids that specific atoms can pull from. 205 // This is a map<atom id, set<uids>> 206 std::map<int32_t, std::set<int32_t>> mPullAtomUids; 207 208 // Packages that specific atoms can be pulled from. 209 std::map<int32_t, std::set<std::string>> mPullAtomPackages; 210 211 // All uids to pull for this atom. NOTE: Does not include the default uids for memory. 212 std::map<int32_t, std::set<int32_t>> mCombinedPullAtomUids; 213 214 // Contains the annotations passed in with StatsdConfig. 215 std::list<std::pair<const int64_t, const int32_t>> mAnnotations; 216 217 bool mShouldPersistHistory; 218 219 // All event tags that are interesting to my metrics. 220 std::set<int> mTagIds; 221 222 // We only store the sp of AtomMatchingTracker, MetricProducer, and ConditionTracker in 223 // MetricsManager. There are relationships between them, and the relationships are denoted by 224 // index instead of pointers. The reasons for this are: (1) the relationship between them are 225 // complicated, so storing index instead of pointers reduces the risk that A holds B's sp, and B 226 // holds A's sp. (2) When we evaluate matcher results, or condition results, we can quickly get 227 // the related results from a cache using the index. 228 229 // Hold all the atom matchers from the config. 230 std::vector<sp<AtomMatchingTracker>> mAllAtomMatchingTrackers; 231 232 // Hold all the conditions from the config. 233 std::vector<sp<ConditionTracker>> mAllConditionTrackers; 234 235 // Hold all metrics from the config. 236 std::vector<sp<MetricProducer>> mAllMetricProducers; 237 238 // Hold all alert trackers. 239 std::vector<sp<AnomalyTracker>> mAllAnomalyTrackers; 240 241 // Hold all periodic alarm trackers. 242 std::vector<sp<AlarmTracker>> mAllPeriodicAlarmTrackers; 243 244 // To make updating configs faster, we map the id of a AtomMatchingTracker, MetricProducer, and 245 // ConditionTracker to its index in the corresponding vector. 246 247 // Maps the id of an atom matching tracker to its index in mAllAtomMatchingTrackers. 248 std::unordered_map<int64_t, int> mAtomMatchingTrackerMap; 249 250 // Maps the id of a condition tracker to its index in mAllConditionTrackers. 251 std::unordered_map<int64_t, int> mConditionTrackerMap; 252 253 // Maps the id of a metric producer to its index in mAllMetricProducers. 254 std::unordered_map<int64_t, int> mMetricProducerMap; 255 256 // To make the log processing more efficient, we want to do as much filtering as possible 257 // before we go into individual trackers and conditions to match. 258 259 // 1st filter: check if the event tag id is in mTagIds. 260 // 2nd filter: if it is, we parse the event because there is at least one member is interested. 261 // then pass to all AtomMatchingTrackers (itself also filter events by ids). 262 // 3nd filter: for AtomMatchingTrackers that matched this event, we pass this event to the 263 // ConditionTrackers and MetricProducers that use this matcher. 264 // 4th filter: for ConditionTrackers that changed value due to this event, we pass 265 // new conditions to metrics that use this condition. 266 267 // The following map is initialized from the statsd_config. 268 269 // Maps from the index of the AtomMatchingTracker to index of MetricProducer. 270 std::unordered_map<int, std::vector<int>> mTrackerToMetricMap; 271 272 // Maps from AtomMatchingTracker to ConditionTracker 273 std::unordered_map<int, std::vector<int>> mTrackerToConditionMap; 274 275 // Maps from ConditionTracker to MetricProducer 276 std::unordered_map<int, std::vector<int>> mConditionToMetricMap; 277 278 // Maps from life span triggering event to MetricProducers. 279 std::unordered_map<int, std::vector<int>> mActivationAtomTrackerToMetricMap; 280 281 // Maps deactivation triggering event to MetricProducers. 282 std::unordered_map<int, std::vector<int>> mDeactivationAtomTrackerToMetricMap; 283 284 // Maps AlertIds to the index of the corresponding AnomalyTracker stored in mAllAnomalyTrackers. 285 // The map is used in LoadMetadata to more efficiently lookup AnomalyTrackers from an AlertId. 286 std::unordered_map<int64_t, int> mAlertTrackerMap; 287 288 std::vector<int> mMetricIndexesWithActivation; 289 290 void initAllowedLogSources(); 291 292 void initPullAtomSources(); 293 294 // Only called on config creation/update to initialize log sources from the config. 295 // Calls initAllowedLogSources and initPullAtomSources. Sets mConfigValid to false on error. 296 void createAllLogSourcesFromConfig(const StatsdConfig& config); 297 298 // Verifies the config meets guardrails and updates statsdStats. 299 // Sets mConfigValid to false on error. Should be called on config creation/update 300 void verifyGuardrailsAndUpdateStatsdStats(); 301 302 // Initializes mIsAlwaysActive and mIsActive. 303 // Should be called on config creation/update. 304 void initializeConfigActiveStatus(); 305 306 // The metrics that don't need to be uploaded or even reported. 307 std::set<int64_t> mNoReportMetricIds; 308 309 // The config is active if any metric in the config is active. 310 bool mIsActive; 311 312 // The config is always active if any metric in the config does not have an activation signal. 313 bool mIsAlwaysActive; 314 315 // Hashes of the States used in this config, keyed by the state id, used in config updates. 316 std::map<int64_t, uint64_t> mStateProtoHashes; 317 318 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensions); 319 FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks); 320 FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByFirstUid); 321 FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByChain); 322 FRIEND_TEST(GaugeMetricE2ePushedTest, TestMultipleFieldsForPushedEvent); 323 FRIEND_TEST(GaugeMetricE2ePulledTest, TestRandomSamplePulledEvents); 324 FRIEND_TEST(GaugeMetricE2ePulledTest, TestRandomSamplePulledEvent_LateAlarm); 325 FRIEND_TEST(GaugeMetricE2ePulledTest, TestRandomSamplePulledEventsWithActivation); 326 FRIEND_TEST(GaugeMetricE2ePulledTest, TestRandomSamplePulledEventsNoCondition); 327 FRIEND_TEST(GaugeMetricE2ePulledTest, TestConditionChangeToTrueSamplePulledEvents); 328 329 FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_single_bucket); 330 FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets); 331 FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written); 332 FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk); 333 FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk); 334 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket); 335 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_partial_bucket); 336 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets); 337 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period); 338 339 FRIEND_TEST(AlarmE2eTest, TestMultipleAlarms); 340 FRIEND_TEST(ConfigTtlE2eTest, TestCountMetric); 341 FRIEND_TEST(ConfigUpdateE2eAbTest, TestConfigTtl); 342 FRIEND_TEST(MetricActivationE2eTest, TestCountMetric); 343 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithOneDeactivation); 344 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoDeactivations); 345 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithSameDeactivation); 346 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoMetricsTwoDeactivations); 347 348 FRIEND_TEST(MetricsManagerTest, TestLogSources); 349 FRIEND_TEST(MetricsManagerTest, TestLogSourcesOnConfigUpdate); 350 351 FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead); 352 FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBoot); 353 FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBootMultipleActivations); 354 FRIEND_TEST(StatsLogProcessorTest, 355 TestActivationOnBootMultipleActivationsDifferentActivationTypes); 356 FRIEND_TEST(StatsLogProcessorTest, TestActivationsPersistAcrossSystemServerRestart); 357 358 FRIEND_TEST(CountMetricE2eTest, TestInitialConditionChanges); 359 FRIEND_TEST(CountMetricE2eTest, TestSlicedState); 360 FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithMap); 361 FRIEND_TEST(CountMetricE2eTest, TestMultipleSlicedStates); 362 FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithPrimaryFields); 363 364 FRIEND_TEST(DurationMetricE2eTest, TestOneBucket); 365 FRIEND_TEST(DurationMetricE2eTest, TestTwoBuckets); 366 FRIEND_TEST(DurationMetricE2eTest, TestWithActivation); 367 FRIEND_TEST(DurationMetricE2eTest, TestWithCondition); 368 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedCondition); 369 FRIEND_TEST(DurationMetricE2eTest, TestWithActivationAndSlicedCondition); 370 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedState); 371 FRIEND_TEST(DurationMetricE2eTest, TestWithConditionAndSlicedState); 372 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStateMapped); 373 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStatePrimaryFieldsSuperset); 374 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStatePrimaryFieldsSubset); 375 FRIEND_TEST(DurationMetricE2eTest, TestUploadThreshold); 376 377 FRIEND_TEST(ValueMetricE2eTest, TestInitialConditionChanges); 378 FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents); 379 FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm); 380 FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_WithActivation); 381 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState); 382 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions); 383 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions); 384 }; 385 386 } // namespace statsd 387 } // namespace os 388 } // namespace android 389