1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <gtest/gtest_prod.h> 20 #include "config/ConfigListener.h" 21 #include "logd/LogEvent.h" 22 #include "metrics/MetricsManager.h" 23 #include "packages/UidMap.h" 24 #include "external/StatsPullerManager.h" 25 26 #include "src/statsd_config.pb.h" 27 #include "src/statsd_metadata.pb.h" 28 29 #include <stdio.h> 30 #include <unordered_map> 31 32 namespace android { 33 namespace os { 34 namespace statsd { 35 36 37 class StatsLogProcessor : public ConfigListener, public virtual PackageInfoListener { 38 public: 39 StatsLogProcessor(const sp<UidMap>& uidMap, const sp<StatsPullerManager>& pullerManager, 40 const sp<AlarmMonitor>& anomalyAlarmMonitor, 41 const sp<AlarmMonitor>& subscriberTriggerAlarmMonitor, 42 const int64_t timeBaseNs, 43 const std::function<bool(const ConfigKey&)>& sendBroadcast, 44 const std::function<bool(const int&, 45 const vector<int64_t>&)>& sendActivationBroadcast); 46 virtual ~StatsLogProcessor(); 47 48 void OnLogEvent(LogEvent* event); 49 50 void OnConfigUpdated(const int64_t timestampNs, const int64_t wallClockNs, const ConfigKey& key, 51 const StatsdConfig& config, bool modularUpdate = true); 52 // For testing only. 53 void OnConfigUpdated(const int64_t timestampNs, const ConfigKey& key, 54 const StatsdConfig& config, bool modularUpdate = true); 55 void OnConfigRemoved(const ConfigKey& key); 56 57 size_t GetMetricsSize(const ConfigKey& key) const; 58 59 void GetActiveConfigs(const int uid, vector<int64_t>& outActiveConfigs); 60 61 void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs, const int64_t wallClockNs, 62 const bool include_current_partial_bucket, const bool erase_data, 63 const DumpReportReason dumpReportReason, const DumpLatency dumpLatency, 64 vector<uint8_t>* outData); 65 void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs, const int64_t wallClockNs, 66 const bool include_current_partial_bucket, const bool erase_data, 67 const DumpReportReason dumpReportReason, const DumpLatency dumpLatency, 68 ProtoOutputStream* proto); 69 // For testing only. 70 void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs, 71 const bool include_current_partial_bucket, const bool erase_data, 72 const DumpReportReason dumpReportReason, const DumpLatency dumpLatency, 73 vector<uint8_t>* outData); 74 75 /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies periodic alarmSet. */ 76 void onPeriodicAlarmFired( 77 const int64_t& timestampNs, 78 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet); 79 80 /* Flushes data to disk. Data on memory will be gone after written to disk. */ 81 void WriteDataToDisk(const DumpReportReason dumpReportReason, const DumpLatency dumpLatency, 82 const int64_t elapsedRealtimeNs, const int64_t wallClockNs); 83 84 /* Persist configs containing metrics with active activations to disk. */ 85 void SaveActiveConfigsToDisk(int64_t currentTimeNs); 86 87 /* Writes the current active status/ttl for all configs and metrics to ProtoOutputStream. */ 88 void WriteActiveConfigsToProtoOutputStream( 89 int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto); 90 91 /* Load configs containing metrics with active activations from disk. */ 92 void LoadActiveConfigsFromDisk(); 93 94 /* Persist metadata for configs and metrics to disk. */ 95 void SaveMetadataToDisk(int64_t currentWallClockTimeNs, int64_t systemElapsedTimeNs); 96 97 /* Writes the statsd metadata for all configs and metrics to StatsMetadataList. */ 98 void WriteMetadataToProto(int64_t currentWallClockTimeNs, 99 int64_t systemElapsedTimeNs, 100 metadata::StatsMetadataList* metadataList); 101 102 /* Load stats metadata for configs and metrics from disk. */ 103 void LoadMetadataFromDisk(int64_t currentWallClockTimeNs, 104 int64_t systemElapsedTimeNs); 105 106 /* Sets the metadata for all configs and metrics */ 107 void SetMetadataState(const metadata::StatsMetadataList& statsMetadataList, 108 int64_t currentWallClockTimeNs, 109 int64_t systemElapsedTimeNs); 110 111 /* Sets the active status/ttl for all configs and metrics to the status in ActiveConfigList. */ 112 void SetConfigsActiveState(const ActiveConfigList& activeConfigList, int64_t currentTimeNs); 113 114 /* Notify all MetricsManagers of app upgrades */ 115 void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid, 116 const int64_t version) override; 117 118 /* Notify all MetricsManagers of app removals */ 119 void notifyAppRemoved(const int64_t& eventTimeNs, const string& apk, const int uid) override; 120 121 /* Notify all MetricsManagers of uid map snapshots received */ 122 void onUidMapReceived(const int64_t& eventTimeNs) override; 123 124 /* Notify all metrics managers of boot completed 125 * This will force a bucket split when the boot is finished. 126 */ 127 void onStatsdInitCompleted(const int64_t& elapsedTimeNs); 128 129 // Reset all configs. 130 void resetConfigs(); 131 getUidMap()132 inline sp<UidMap> getUidMap() { 133 return mUidMap; 134 } 135 136 void dumpStates(int outFd, bool verbose); 137 138 void informPullAlarmFired(const int64_t timestampNs); 139 140 int64_t getLastReportTimeNs(const ConfigKey& key); 141 setPrintLogs(bool enabled)142 inline void setPrintLogs(bool enabled) { 143 std::lock_guard<std::mutex> lock(mMetricsMutex); 144 mPrintAllLogs = enabled; 145 } 146 147 // Add a specific config key to the possible configs to dump ASAP. 148 void noteOnDiskData(const ConfigKey& key); 149 150 void setAnomalyAlarm(const int64_t timeMillis); 151 152 void cancelAnomalyAlarm(); 153 154 private: 155 // For testing only. getAnomalyAlarmMonitor()156 inline sp<AlarmMonitor> getAnomalyAlarmMonitor() const { 157 return mAnomalyAlarmMonitor; 158 } 159 getPeriodicAlarmMonitor()160 inline sp<AlarmMonitor> getPeriodicAlarmMonitor() const { 161 return mPeriodicAlarmMonitor; 162 } 163 164 mutable mutex mMetricsMutex; 165 166 // Guards mNextAnomalyAlarmTime. A separate mutex is needed because alarms are set/cancelled 167 // in the onLogEvent code path, which is locked by mMetricsMutex. 168 // DO NOT acquire mMetricsMutex while holding mAnomalyAlarmMutex. This can lead to a deadlock. 169 mutable mutex mAnomalyAlarmMutex; 170 171 std::unordered_map<ConfigKey, sp<MetricsManager>> mMetricsManagers; 172 173 std::unordered_map<ConfigKey, int64_t> mLastBroadcastTimes; 174 175 // Last time we sent a broadcast to this uid that the active configs had changed. 176 std::unordered_map<int, int64_t> mLastActivationBroadcastTimes; 177 178 // Tracks when we last checked the bytes consumed for each config key. 179 std::unordered_map<ConfigKey, int64_t> mLastByteSizeTimes; 180 181 // Tracks which config keys has metric reports on disk 182 std::set<ConfigKey> mOnDiskDataConfigs; 183 184 sp<UidMap> mUidMap; // Reference to the UidMap to lookup app name and version for each uid. 185 186 sp<StatsPullerManager> mPullerManager; // Reference to StatsPullerManager 187 188 sp<AlarmMonitor> mAnomalyAlarmMonitor; 189 190 sp<AlarmMonitor> mPeriodicAlarmMonitor; 191 192 void OnLogEvent(LogEvent* event, int64_t elapsedRealtimeNs); 193 194 void resetIfConfigTtlExpiredLocked(const int64_t eventTimeNs); 195 196 void OnConfigUpdatedLocked(const int64_t currentTimestampNs, const ConfigKey& key, 197 const StatsdConfig& config, bool modularUpdate); 198 199 void GetActiveConfigsLocked(const int uid, vector<int64_t>& outActiveConfigs); 200 201 void WriteActiveConfigsToProtoOutputStreamLocked( 202 int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto); 203 204 void SetConfigsActiveStateLocked(const ActiveConfigList& activeConfigList, 205 int64_t currentTimeNs); 206 207 void SetMetadataStateLocked(const metadata::StatsMetadataList& statsMetadataList, 208 int64_t currentWallClockTimeNs, 209 int64_t systemElapsedTimeNs); 210 211 void WriteMetadataToProtoLocked(int64_t currentWallClockTimeNs, 212 int64_t systemElapsedTimeNs, 213 metadata::StatsMetadataList* metadataList); 214 215 void WriteDataToDiskLocked(const DumpReportReason dumpReportReason, 216 const DumpLatency dumpLatency, const int64_t elapsedRealtimeNs, 217 const int64_t wallClockNs); 218 219 void WriteDataToDiskLocked(const ConfigKey& key, const int64_t timestampNs, 220 const int64_t wallClockNs, const DumpReportReason dumpReportReason, 221 const DumpLatency dumpLatency); 222 223 void onConfigMetricsReportLocked( 224 const ConfigKey& key, const int64_t dumpTimeStampNs, const int64_t wallClockNs, 225 const bool include_current_partial_bucket, const bool erase_data, 226 const DumpReportReason dumpReportReason, const DumpLatency dumpLatency, 227 /*if dataSavedToDisk is true, it indicates the caller will write the data to disk 228 (e.g., before reboot). So no need to further persist local history.*/ 229 const bool dataSavedToDisk, vector<uint8_t>* proto); 230 231 /* Check if we should send a broadcast if approaching memory limits and if we're over, we 232 * actually delete the data. */ 233 void flushIfNecessaryLocked(const ConfigKey& key, MetricsManager& metricsManager); 234 235 // Maps the isolated uid in the log event to host uid if the log event contains uid fields. 236 void mapIsolatedUidToHostUidIfNecessaryLocked(LogEvent* event) const; 237 238 // Handler over the isolated uid change event. 239 void onIsolatedUidChangedEventLocked(const LogEvent& event); 240 241 // Handler over the binary push state changed event. 242 void onBinaryPushStateChangedEventLocked(LogEvent* event); 243 244 // Handler over the watchdog rollback occurred event. 245 void onWatchdogRollbackOccurredLocked(LogEvent* event); 246 247 // Updates train info on disk based on binary push state changed info and 248 // write disk info into parameters. 249 void getAndUpdateTrainInfoOnDisk(bool is_rollback, InstallTrainInfo* trainInfoIn); 250 251 // Gets experiment ids on disk for associated train and updates them 252 // depending on rollback type. Then writes them back to disk and returns 253 // them. 254 std::vector<int64_t> processWatchdogRollbackOccurred(const int32_t rollbackTypeIn, 255 const string& packageName); 256 257 // Reset all configs. 258 void resetConfigsLocked(const int64_t timestampNs); 259 // Reset the specified configs. 260 void resetConfigsLocked(const int64_t timestampNs, const std::vector<ConfigKey>& configs); 261 262 // An anomaly alarm should have fired. 263 // Check with anomaly alarm manager to find the alarms and process the result. 264 void informAnomalyAlarmFiredLocked(const int64_t elapsedTimeMillis); 265 266 /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies anomaly alarmSet. */ 267 void processFiredAnomalyAlarmsLocked( 268 const int64_t& timestampNs, 269 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet); 270 271 // Function used to send a broadcast so that receiver for the config key can call getData 272 // to retrieve the stored data. 273 std::function<bool(const ConfigKey& key)> mSendBroadcast; 274 275 // Function used to send a broadcast so that receiver can be notified of which configs 276 // are currently active. 277 std::function<bool(const int& uid, const vector<int64_t>& configIds)> mSendActivationBroadcast; 278 279 const int64_t mTimeBaseNs; 280 281 // Largest timestamp of the events that we have processed. 282 int64_t mLargestTimestampSeen = 0; 283 284 int64_t mLastTimestampSeen = 0; 285 286 int64_t mLastPullerCacheClearTimeSec = 0; 287 288 // Last time we wrote data to disk. 289 int64_t mLastWriteTimeNs = 0; 290 291 // Last time we wrote active metrics to disk. 292 int64_t mLastActiveMetricsWriteNs = 0; 293 294 //Last time we wrote metadata to disk. 295 int64_t mLastMetadataWriteNs = 0; 296 297 // The time for the next anomaly alarm for alerts. 298 int64_t mNextAnomalyAlarmTime = 0; 299 300 bool mPrintAllLogs = false; 301 302 FRIEND_TEST(StatsLogProcessorTest, TestOutOfOrderLogs); 303 FRIEND_TEST(StatsLogProcessorTest, TestRateLimitByteSize); 304 FRIEND_TEST(StatsLogProcessorTest, TestRateLimitBroadcast); 305 FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge); 306 FRIEND_TEST(StatsLogProcessorTest, InvalidConfigRemoved); 307 FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead); 308 FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBoot); 309 FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBootMultipleActivations); 310 FRIEND_TEST(StatsLogProcessorTest, 311 TestActivationOnBootMultipleActivationsDifferentActivationTypes); 312 FRIEND_TEST(StatsLogProcessorTest, TestActivationsPersistAcrossSystemServerRestart); 313 314 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration1); 315 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration2); 316 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration3); 317 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration1); 318 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration2); 319 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration3); 320 FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks1); 321 FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks2); 322 FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByFirstUid); 323 FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByChain); 324 FRIEND_TEST(GaugeMetricE2ePushedTest, TestMultipleFieldsForPushedEvent); 325 FRIEND_TEST(GaugeMetricE2ePushedTest, TestRepeatedFieldsForPushedEvent); 326 FRIEND_TEST(GaugeMetricE2ePulledTest, TestRandomSamplePulledEvents); 327 FRIEND_TEST(GaugeMetricE2ePulledTest, TestRandomSamplePulledEvent_LateAlarm); 328 FRIEND_TEST(GaugeMetricE2ePulledTest, TestRandomSamplePulledEventsWithActivation); 329 FRIEND_TEST(GaugeMetricE2ePulledTest, TestRandomSamplePulledEventsNoCondition); 330 FRIEND_TEST(GaugeMetricE2ePulledTest, TestConditionChangeToTrueSamplePulledEvents); 331 332 FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_single_bucket); 333 FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets); 334 FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written); 335 FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk); 336 FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk); 337 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket); 338 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_partial_bucket); 339 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets); 340 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period); 341 342 FRIEND_TEST(AlarmE2eTest, TestMultipleAlarms); 343 FRIEND_TEST(ConfigTtlE2eTest, TestCountMetric); 344 FRIEND_TEST(MetricActivationE2eTest, TestCountMetric); 345 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithOneDeactivation); 346 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoDeactivations); 347 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithSameDeactivation); 348 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoMetricsTwoDeactivations); 349 350 FRIEND_TEST(ConfigUpdateE2eTest, TestAlarms); 351 FRIEND_TEST(ConfigUpdateE2eTest, TestGaugeMetric); 352 FRIEND_TEST(ConfigUpdateE2eTest, TestValueMetric); 353 FRIEND_TEST(ConfigUpdateE2eTest, TestAnomalyDurationMetric); 354 FRIEND_TEST(ConfigUpdateE2eAbTest, TestHashStrings); 355 FRIEND_TEST(ConfigUpdateE2eAbTest, TestUidMapVersionStringInstaller); 356 FRIEND_TEST(ConfigUpdateE2eAbTest, TestConfigTtl); 357 358 FRIEND_TEST(CountMetricE2eTest, TestInitialConditionChanges); 359 FRIEND_TEST(CountMetricE2eTest, TestSlicedState); 360 FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithMap); 361 FRIEND_TEST(CountMetricE2eTest, TestMultipleSlicedStates); 362 FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithPrimaryFields); 363 FRIEND_TEST(CountMetricE2eTest, TestRepeatedFieldsAndEmptyArrays); 364 365 FRIEND_TEST(DurationMetricE2eTest, TestOneBucket); 366 FRIEND_TEST(DurationMetricE2eTest, TestTwoBuckets); 367 FRIEND_TEST(DurationMetricE2eTest, TestWithActivation); 368 FRIEND_TEST(DurationMetricE2eTest, TestWithCondition); 369 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedCondition); 370 FRIEND_TEST(DurationMetricE2eTest, TestWithActivationAndSlicedCondition); 371 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedState); 372 FRIEND_TEST(DurationMetricE2eTest, TestWithConditionAndSlicedState); 373 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStateMapped); 374 FRIEND_TEST(DurationMetricE2eTest, TestSlicedStatePrimaryFieldsNotSubsetDimInWhat); 375 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStatePrimaryFieldsSubset); 376 FRIEND_TEST(DurationMetricE2eTest, TestUploadThreshold); 377 FRIEND_TEST(DurationMetricE2eTest, TestConditionOnRepeatedEnumField); 378 379 FRIEND_TEST(ValueMetricE2eTest, TestInitialConditionChanges); 380 FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents); 381 FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm); 382 FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_WithActivation); 383 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState); 384 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions); 385 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions); 386 FRIEND_TEST(ValueMetricE2eTest, TestInitWithValueFieldPositionALL); 387 388 FRIEND_TEST(KllMetricE2eTest, TestInitWithKllFieldPositionALL); 389 }; 390 391 } // namespace statsd 392 } // namespace os 393 } // namespace android 394