1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <gtest/gtest_prod.h> 20 #include "config/ConfigListener.h" 21 #include "logd/LogEvent.h" 22 #include "metrics/MetricsManager.h" 23 #include "packages/UidMap.h" 24 #include "external/StatsPullerManager.h" 25 26 #include "src/statsd_config.pb.h" 27 #include "src/statsd_metadata.pb.h" 28 29 #include <stdio.h> 30 #include <unordered_map> 31 32 namespace android { 33 namespace os { 34 namespace statsd { 35 36 37 class StatsLogProcessor : public ConfigListener, public virtual PackageInfoListener { 38 public: 39 StatsLogProcessor(const sp<UidMap>& uidMap, const sp<StatsPullerManager>& pullerManager, 40 const sp<AlarmMonitor>& anomalyAlarmMonitor, 41 const sp<AlarmMonitor>& subscriberTriggerAlarmMonitor, 42 const int64_t timeBaseNs, 43 const std::function<bool(const ConfigKey&)>& sendBroadcast, 44 const std::function<bool(const int&, 45 const vector<int64_t>&)>& sendActivationBroadcast); 46 virtual ~StatsLogProcessor(); 47 48 void OnLogEvent(LogEvent* event); 49 50 void OnConfigUpdated(const int64_t timestampNs, const ConfigKey& key, 51 const StatsdConfig& config, bool modularUpdate = true); 52 void OnConfigRemoved(const ConfigKey& key); 53 54 size_t GetMetricsSize(const ConfigKey& key) const; 55 56 void GetActiveConfigs(const int uid, vector<int64_t>& outActiveConfigs); 57 58 void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs, 59 const bool include_current_partial_bucket, const bool erase_data, 60 const DumpReportReason dumpReportReason, 61 const DumpLatency dumpLatency, 62 vector<uint8_t>* outData); 63 void onDumpReport(const ConfigKey& key, const int64_t dumpTimeNs, 64 const bool include_current_partial_bucket, const bool erase_data, 65 const DumpReportReason dumpReportReason, 66 const DumpLatency dumpLatency, 67 ProtoOutputStream* proto); 68 69 /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies periodic alarmSet. */ 70 void onPeriodicAlarmFired( 71 const int64_t& timestampNs, 72 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet); 73 74 /* Flushes data to disk. Data on memory will be gone after written to disk. */ 75 void WriteDataToDisk(const DumpReportReason dumpReportReason, const DumpLatency dumpLatency, 76 const int64_t elapsedRealtimeNs); 77 78 /* Persist configs containing metrics with active activations to disk. */ 79 void SaveActiveConfigsToDisk(int64_t currentTimeNs); 80 81 /* Writes the current active status/ttl for all configs and metrics to ProtoOutputStream. */ 82 void WriteActiveConfigsToProtoOutputStream( 83 int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto); 84 85 /* Load configs containing metrics with active activations from disk. */ 86 void LoadActiveConfigsFromDisk(); 87 88 /* Persist metadata for configs and metrics to disk. */ 89 void SaveMetadataToDisk(int64_t currentWallClockTimeNs, int64_t systemElapsedTimeNs); 90 91 /* Writes the statsd metadata for all configs and metrics to StatsMetadataList. */ 92 void WriteMetadataToProto(int64_t currentWallClockTimeNs, 93 int64_t systemElapsedTimeNs, 94 metadata::StatsMetadataList* metadataList); 95 96 /* Load stats metadata for configs and metrics from disk. */ 97 void LoadMetadataFromDisk(int64_t currentWallClockTimeNs, 98 int64_t systemElapsedTimeNs); 99 100 /* Sets the metadata for all configs and metrics */ 101 void SetMetadataState(const metadata::StatsMetadataList& statsMetadataList, 102 int64_t currentWallClockTimeNs, 103 int64_t systemElapsedTimeNs); 104 105 /* Sets the active status/ttl for all configs and metrics to the status in ActiveConfigList. */ 106 void SetConfigsActiveState(const ActiveConfigList& activeConfigList, int64_t currentTimeNs); 107 108 /* Notify all MetricsManagers of app upgrades */ 109 void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid, 110 const int64_t version) override; 111 112 /* Notify all MetricsManagers of app removals */ 113 void notifyAppRemoved(const int64_t& eventTimeNs, const string& apk, const int uid) override; 114 115 /* Notify all MetricsManagers of uid map snapshots received */ 116 void onUidMapReceived(const int64_t& eventTimeNs) override; 117 118 /* Notify all metrics managers of boot completed 119 * This will force a bucket split when the boot is finished. 120 */ 121 void onStatsdInitCompleted(const int64_t& elapsedTimeNs); 122 123 // Reset all configs. 124 void resetConfigs(); 125 getUidMap()126 inline sp<UidMap> getUidMap() { 127 return mUidMap; 128 } 129 130 void dumpStates(int outFd, bool verbose); 131 132 void informPullAlarmFired(const int64_t timestampNs); 133 134 int64_t getLastReportTimeNs(const ConfigKey& key); 135 setPrintLogs(bool enabled)136 inline void setPrintLogs(bool enabled) { 137 std::lock_guard<std::mutex> lock(mMetricsMutex); 138 mPrintAllLogs = enabled; 139 } 140 141 // Add a specific config key to the possible configs to dump ASAP. 142 void noteOnDiskData(const ConfigKey& key); 143 144 void setAnomalyAlarm(const int64_t timeMillis); 145 146 void cancelAnomalyAlarm(); 147 148 private: 149 // For testing only. getAnomalyAlarmMonitor()150 inline sp<AlarmMonitor> getAnomalyAlarmMonitor() const { 151 return mAnomalyAlarmMonitor; 152 } 153 getPeriodicAlarmMonitor()154 inline sp<AlarmMonitor> getPeriodicAlarmMonitor() const { 155 return mPeriodicAlarmMonitor; 156 } 157 158 mutable mutex mMetricsMutex; 159 160 // Guards mNextAnomalyAlarmTime. A separate mutex is needed because alarms are set/cancelled 161 // in the onLogEvent code path, which is locked by mMetricsMutex. 162 // DO NOT acquire mMetricsMutex while holding mAnomalyAlarmMutex. This can lead to a deadlock. 163 mutable mutex mAnomalyAlarmMutex; 164 165 std::unordered_map<ConfigKey, sp<MetricsManager>> mMetricsManagers; 166 167 std::unordered_map<ConfigKey, int64_t> mLastBroadcastTimes; 168 169 // Last time we sent a broadcast to this uid that the active configs had changed. 170 std::unordered_map<int, int64_t> mLastActivationBroadcastTimes; 171 172 // Tracks when we last checked the bytes consumed for each config key. 173 std::unordered_map<ConfigKey, int64_t> mLastByteSizeTimes; 174 175 // Tracks which config keys has metric reports on disk 176 std::set<ConfigKey> mOnDiskDataConfigs; 177 178 sp<UidMap> mUidMap; // Reference to the UidMap to lookup app name and version for each uid. 179 180 sp<StatsPullerManager> mPullerManager; // Reference to StatsPullerManager 181 182 sp<AlarmMonitor> mAnomalyAlarmMonitor; 183 184 sp<AlarmMonitor> mPeriodicAlarmMonitor; 185 186 void OnLogEvent(LogEvent* event, int64_t elapsedRealtimeNs); 187 188 void resetIfConfigTtlExpiredLocked(const int64_t eventTimeNs); 189 190 void OnConfigUpdatedLocked(const int64_t currentTimestampNs, const ConfigKey& key, 191 const StatsdConfig& config, bool modularUpdate); 192 193 void GetActiveConfigsLocked(const int uid, vector<int64_t>& outActiveConfigs); 194 195 void WriteActiveConfigsToProtoOutputStreamLocked( 196 int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto); 197 198 void SetConfigsActiveStateLocked(const ActiveConfigList& activeConfigList, 199 int64_t currentTimeNs); 200 201 void SetMetadataStateLocked(const metadata::StatsMetadataList& statsMetadataList, 202 int64_t currentWallClockTimeNs, 203 int64_t systemElapsedTimeNs); 204 205 void WriteMetadataToProtoLocked(int64_t currentWallClockTimeNs, 206 int64_t systemElapsedTimeNs, 207 metadata::StatsMetadataList* metadataList); 208 209 void WriteDataToDiskLocked(const DumpReportReason dumpReportReason, 210 const DumpLatency dumpLatency, const int64_t elapsedRealtimeNs); 211 212 void WriteDataToDiskLocked(const ConfigKey& key, const int64_t timestampNs, 213 const DumpReportReason dumpReportReason, 214 const DumpLatency dumpLatency); 215 216 void onConfigMetricsReportLocked( 217 const ConfigKey& key, const int64_t dumpTimeStampNs, 218 const bool include_current_partial_bucket, const bool erase_data, 219 const DumpReportReason dumpReportReason, const DumpLatency dumpLatency, 220 /*if dataSavedToDisk is true, it indicates the caller will write the data to disk 221 (e.g., before reboot). So no need to further persist local history.*/ 222 const bool dataSavedToDisk, vector<uint8_t>* proto); 223 224 /* Check if we should send a broadcast if approaching memory limits and if we're over, we 225 * actually delete the data. */ 226 void flushIfNecessaryLocked(const ConfigKey& key, MetricsManager& metricsManager); 227 228 // Maps the isolated uid in the log event to host uid if the log event contains uid fields. 229 void mapIsolatedUidToHostUidIfNecessaryLocked(LogEvent* event) const; 230 231 // Handler over the isolated uid change event. 232 void onIsolatedUidChangedEventLocked(const LogEvent& event); 233 234 // Handler over the binary push state changed event. 235 void onBinaryPushStateChangedEventLocked(LogEvent* event); 236 237 // Handler over the watchdog rollback occurred event. 238 void onWatchdogRollbackOccurredLocked(LogEvent* event); 239 240 // Updates train info on disk based on binary push state changed info and 241 // write disk info into parameters. 242 void getAndUpdateTrainInfoOnDisk(bool is_rollback, InstallTrainInfo* trainInfoIn); 243 244 // Gets experiment ids on disk for associated train and updates them 245 // depending on rollback type. Then writes them back to disk and returns 246 // them. 247 std::vector<int64_t> processWatchdogRollbackOccurred(const int32_t rollbackTypeIn, 248 const string& packageName); 249 250 // Reset all configs. 251 void resetConfigsLocked(const int64_t timestampNs); 252 // Reset the specified configs. 253 void resetConfigsLocked(const int64_t timestampNs, const std::vector<ConfigKey>& configs); 254 255 // An anomaly alarm should have fired. 256 // Check with anomaly alarm manager to find the alarms and process the result. 257 void informAnomalyAlarmFiredLocked(const int64_t elapsedTimeMillis); 258 259 /* Tells MetricsManager that the alarms in alarmSet have fired. Modifies anomaly alarmSet. */ 260 void processFiredAnomalyAlarmsLocked( 261 const int64_t& timestampNs, 262 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet); 263 264 // Function used to send a broadcast so that receiver for the config key can call getData 265 // to retrieve the stored data. 266 std::function<bool(const ConfigKey& key)> mSendBroadcast; 267 268 // Function used to send a broadcast so that receiver can be notified of which configs 269 // are currently active. 270 std::function<bool(const int& uid, const vector<int64_t>& configIds)> mSendActivationBroadcast; 271 272 const int64_t mTimeBaseNs; 273 274 // Largest timestamp of the events that we have processed. 275 int64_t mLargestTimestampSeen = 0; 276 277 int64_t mLastTimestampSeen = 0; 278 279 int64_t mLastPullerCacheClearTimeSec = 0; 280 281 // Last time we wrote data to disk. 282 int64_t mLastWriteTimeNs = 0; 283 284 // Last time we wrote active metrics to disk. 285 int64_t mLastActiveMetricsWriteNs = 0; 286 287 //Last time we wrote metadata to disk. 288 int64_t mLastMetadataWriteNs = 0; 289 290 // The time for the next anomaly alarm for alerts. 291 int64_t mNextAnomalyAlarmTime = 0; 292 293 bool mPrintAllLogs = false; 294 295 FRIEND_TEST(StatsLogProcessorTest, TestOutOfOrderLogs); 296 FRIEND_TEST(StatsLogProcessorTest, TestRateLimitByteSize); 297 FRIEND_TEST(StatsLogProcessorTest, TestRateLimitBroadcast); 298 FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge); 299 FRIEND_TEST(StatsLogProcessorTest, InvalidConfigRemoved); 300 FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead); 301 FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBoot); 302 FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBootMultipleActivations); 303 FRIEND_TEST(StatsLogProcessorTest, 304 TestActivationOnBootMultipleActivationsDifferentActivationTypes); 305 FRIEND_TEST(StatsLogProcessorTest, TestActivationsPersistAcrossSystemServerRestart); 306 307 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration1); 308 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration2); 309 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration3); 310 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration1); 311 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration2); 312 FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration3); 313 FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks1); 314 FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks2); 315 FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByFirstUid); 316 FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByChain); 317 FRIEND_TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent); 318 FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents); 319 FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm); 320 FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEventsWithActivation); 321 FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEventsNoCondition); 322 FRIEND_TEST(GaugeMetricE2eTest, TestConditionChangeToTrueSamplePulledEvents); 323 324 FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_single_bucket); 325 FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets); 326 FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written); 327 FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk); 328 FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk); 329 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket); 330 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_partial_bucket); 331 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets); 332 FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period); 333 334 FRIEND_TEST(AlarmE2eTest, TestMultipleAlarms); 335 FRIEND_TEST(ConfigTtlE2eTest, TestCountMetric); 336 FRIEND_TEST(MetricActivationE2eTest, TestCountMetric); 337 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithOneDeactivation); 338 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoDeactivations); 339 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithSameDeactivation); 340 FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoMetricsTwoDeactivations); 341 342 FRIEND_TEST(ConfigUpdateE2eTest, TestAlarms); 343 FRIEND_TEST(ConfigUpdateE2eTest, TestGaugeMetric); 344 FRIEND_TEST(ConfigUpdateE2eTest, TestValueMetric); 345 FRIEND_TEST(ConfigUpdateE2eTest, TestAnomalyDurationMetric); 346 FRIEND_TEST(ConfigUpdateE2eAbTest, TestHashStrings); 347 FRIEND_TEST(ConfigUpdateE2eAbTest, TestUidMapVersionStringInstaller); 348 FRIEND_TEST(ConfigUpdateE2eAbTest, TestConfigTtl); 349 350 FRIEND_TEST(CountMetricE2eTest, TestInitialConditionChanges); 351 FRIEND_TEST(CountMetricE2eTest, TestSlicedState); 352 FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithMap); 353 FRIEND_TEST(CountMetricE2eTest, TestMultipleSlicedStates); 354 FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithPrimaryFields); 355 356 FRIEND_TEST(DurationMetricE2eTest, TestOneBucket); 357 FRIEND_TEST(DurationMetricE2eTest, TestTwoBuckets); 358 FRIEND_TEST(DurationMetricE2eTest, TestWithActivation); 359 FRIEND_TEST(DurationMetricE2eTest, TestWithCondition); 360 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedCondition); 361 FRIEND_TEST(DurationMetricE2eTest, TestWithActivationAndSlicedCondition); 362 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedState); 363 FRIEND_TEST(DurationMetricE2eTest, TestWithConditionAndSlicedState); 364 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStateMapped); 365 FRIEND_TEST(DurationMetricE2eTest, TestSlicedStatePrimaryFieldsNotSubsetDimInWhat); 366 FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStatePrimaryFieldsSubset); 367 FRIEND_TEST(DurationMetricE2eTest, TestUploadThreshold); 368 369 FRIEND_TEST(ValueMetricE2eTest, TestInitialConditionChanges); 370 FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents); 371 FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm); 372 FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_WithActivation); 373 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState); 374 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions); 375 FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions); 376 }; 377 378 } // namespace statsd 379 } // namespace os 380 } // namespace android 381