xref: /packages/modules/StatsD/statsd/src/guardrail/StatsdStats.h

/*
 * Copyright 2017, The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#pragma once

#include <gtest/gtest_prod.h>
#include <log/log_time.h>
#include <src/guardrail/stats_log_enums.pb.h>

#include <limits>
#include <list>
#include <mutex>
#include <string>
#include <unordered_map>
#include <vector>

#include "LoggingRate.h"
#include "config/ConfigKey.h"
#include "logd/logevent_util.h"

namespace android {
namespace os {
namespace statsd {

struct InvalidConfigReason {
    InvalidConfigReasonEnum reason;
    std::optional<int64_t> metricId;
    std::optional<int64_t> stateId;
    std::optional<int64_t> alertId;
    std::optional<int64_t> alarmId;
    std::optional<int64_t> subscriptionId;
    std::vector<int64_t> matcherIds;
    std::vector<int64_t> conditionIds;
    InvalidConfigReason(){};
    InvalidConfigReason(InvalidConfigReasonEnum reason) : reason(reason){};
    InvalidConfigReason(InvalidConfigReasonEnum reason, int64_t metricId)
        : reason(reason), metricId(metricId){};
    bool operator==(const InvalidConfigReason& other) const {
        return (this->reason == other.reason) && (this->metricId == other.metricId) &&
               (this->stateId == other.stateId) && (this->alertId == other.alertId) &&
               (this->alarmId == other.alarmId) && (this->subscriptionId == other.subscriptionId) &&
               (this->matcherIds == other.matcherIds) && (this->conditionIds == other.conditionIds);
    }

    // For better failure messages in statsd_test
    friend void PrintTo(const InvalidConfigReason& obj, std::ostream* os);
};

typedef struct {
    int64_t insertError = 0;
    int64_t tableCreationError = 0;
    int64_t tableDeletionError = 0;
    std::list<int64_t> flushLatencyNs;
    int64_t categoryChangedCount = 0;
} RestrictedMetricStats;

struct DumpReportStats {
    DumpReportStats(int32_t dumpReportSec, int32_t dumpReportSize, int32_t reportNumber)
        : mDumpReportTimeSec(dumpReportSec),
          mDumpReportSizeBytes(dumpReportSize),
          mDumpReportNumber(reportNumber) {
    }
    int32_t mDumpReportTimeSec = 0;
    int32_t mDumpReportSizeBytes = 0;
    int32_t mDumpReportNumber = 0;
};

struct ConfigStats {
    int32_t uid;
    int64_t id;
    int32_t creation_time_sec;
    int32_t deletion_time_sec = 0;
    int32_t reset_time_sec = 0;
    int32_t metric_count;
    int32_t condition_count;
    int32_t matcher_count;
    int32_t alert_count;
    bool is_valid;
    bool device_info_table_creation_failed = false;
    int32_t db_corrupted_count = 0;
    int32_t db_deletion_stat_failed = 0;
    int32_t db_deletion_size_exceeded_limit = 0;
    int32_t db_deletion_config_invalid = 0;
    int32_t db_deletion_too_old = 0;
    int32_t db_deletion_config_removed = 0;
    int32_t db_deletion_config_updated = 0;
    // Stores the number of ConfigMetadataProvider promotion failures
    int32_t config_metadata_provider_promote_failure = 0;

    // Stores reasons for why config is valid or not
    std::optional<InvalidConfigReason> reason;

    std::list<int32_t> broadcast_sent_time_sec;

    // Times at which this config is activated.
    std::list<int32_t> activation_time_sec;

    // Times at which this config is deactivated.
    std::list<int32_t> deactivation_time_sec;

    std::list<int32_t> data_drop_time_sec;
    // Number of bytes dropped at corresponding time.
    std::list<int64_t> data_drop_bytes;

    std::list<DumpReportStats> dump_report_stats;

    // Stores how many times a matcher have been matched. The map size is capped by kMaxConfigCount.
    std::unordered_map<int64_t, int> matcher_stats;

    // Stores the number of output tuple of condition trackers when it's bigger than
    // kDimensionKeySizeSoftLimit. When you see the number is kDimensionKeySizeHardLimit +1,
    // it means some data has been dropped. The map size is capped by kMaxConfigCount.
    std::map<const int64_t, int> condition_stats;

    // Stores the number of output tuple of metric producers when it's bigger than
    // kDimensionKeySizeSoftLimit. When you see the number is kDimensionKeySizeHardLimit +1,
    // it means some data has been dropped. The map size is capped by kMaxConfigCount.
    std::map<const int64_t, int> metric_stats;

    // Stores the max number of output tuple of dimensions in condition across dimensions in what
    // when it's bigger than kDimensionKeySizeSoftLimit. When you see the number is
    // kDimensionKeySizeHardLimit +1, it means some data has been dropped. The map size is capped by
    // kMaxConfigCount.
    std::map<const int64_t, int> metric_dimension_in_condition_stats;

    // Stores the number of times an anomaly detection alert has been declared.
    // The map size is capped by kMaxConfigCount.
    std::map<const int64_t, int> alert_stats;

    // Stores the config ID for each sub-config used.
    std::list<std::pair<const int64_t, const int32_t>> annotations;

    // Maps metric ID of restricted metric to its stats.
    std::map<int64_t, RestrictedMetricStats> restricted_metric_stats;

    std::list<int64_t> total_flush_latency_ns;

    // Stores the last 20 timestamps for computing sqlite db size.
    std::list<int64_t> total_db_size_timestamps;

    // Stores the last 20 sizes of the sqlite db.
    std::list<int64_t> total_db_sizes;
};

struct UidMapStats {
    int32_t changes = 0;
    int32_t bytes_used = 0;
    int32_t dropped_changes = 0;
    int32_t deleted_apps = 0;
};

struct SubscriptionStats {
    int32_t pushed_atom_count = 0;
    int32_t pulled_atom_count = 0;
    int32_t start_time_sec = 0;
    int32_t end_time_sec = 0;
    int32_t flush_count = 0;
};

// Keeps track of stats of statsd.
// Single instance shared across the process. All public methods are thread safe.
class StatsdStats {
public:
    static StatsdStats& getInstance();
    ~StatsdStats(){};

    const static int kDimensionKeySizeSoftLimit = 500;
    static constexpr int kDimensionKeySizeHardLimit = 800;
    static constexpr int kDimensionKeySizeHardLimitMin = 800;
    static constexpr int kDimensionKeySizeHardLimitMax = 3000;

    // Per atom dimension key size limit
    static const std::map<int, std::pair<size_t, size_t>> kAtomDimensionKeySizeLimitMap;

    const static int kMaxConfigCountPerUid = 20;
    const static int kMaxAlertCountPerConfig = 200;
    const static int kMaxConditionCountPerConfig = 500;
    const static int kMaxMetricCountPerConfig = 3000;
    const static int kMaxMatcherCountPerConfig = 3500;

    // The max number of old config stats we keep.
    const static int kMaxIceBoxSize = 20;

    const static int kMaxLoggerErrors = 20;

    const static int kMaxSystemServerRestarts = 20;

    const static int kMaxTimestampCount = 20;

    const static int kMaxLogSourceCount = 150;

    const static int kMaxPullAtomPackages = 100;

    const static int kMaxRestrictedMetricQueryCount = 20;

    const static int kMaxRestrictedMetricFlushLatencyCount = 20;

    const static int kMaxRestrictedConfigFlushLatencyCount = 20;

    const static int kMaxRestrictedConfigDbSizeCount = 20;

    // Max memory allowed for storing metrics per configuration. If this limit is exceeded, statsd
    // drops the metrics data in memory.
    static const size_t kDefaultMaxMetricsBytesPerConfig = 2 * 1024 * 1024;

    // Hard limit for custom memory allowed for storing metrics per configuration.
    static const size_t kHardMaxMetricsBytesPerConfig = 20 * 1024 * 1024;

    // Max memory allowed for storing metrics per configuration before triggering a intent to fetch
    // data.
    static const size_t kHardMaxTriggerGetDataBytes = 10 * 1024 * 1024;

    // Soft memory limit per configuration. Once this limit is exceeded, we begin notifying the
    // data subscriber that it's time to call getData.
    static const size_t kDefaultBytesPerConfigTriggerGetData = 192 * 1024;

    // Soft memory limit per restricted configuration. Once this limit is exceeded,
    // we begin flush in-memory restricted metrics to database.
    static const size_t kBytesPerRestrictedConfigTriggerFlush = 25 * 1024;

    // Cap the UID map's memory usage to this. This should be fairly high since the UID information
    // is critical for understanding the metrics.
    const static size_t kMaxBytesUsedUidMap = 50 * 1024;

    // The number of deleted apps that are stored in the uid map.
    const static int kMaxDeletedAppsInUidMap = 100;

    /* Minimum period between two broadcasts in nanoseconds. */
    static const int64_t kMinBroadcastPeriodNs = 60 * NS_PER_SEC;

    /* Min period between two checks of byte size per config key in nanoseconds. */
    static const int64_t kMinByteSizeCheckPeriodNs = 1 * 60 * NS_PER_SEC;

    // Min period between two checks of byte size per config key in nanoseconds for V2 memory
    // calculations.
    static const int64_t kMinByteSizeV2CheckPeriodNs = 5 * 60 * NS_PER_SEC;

    /* Min period between two checks of restricted metrics TTLs. */
    static const int64_t kMinTtlCheckPeriodNs = 60 * 60 * NS_PER_SEC;

    /* Min period between two flush operations of restricted metrics. */
    static const int64_t kMinFlushRestrictedPeriodNs = 60 * 60 * NS_PER_SEC;

    /* Min period between two db guardrail check operations of restricted metrics. */
    static const int64_t kMinDbGuardrailEnforcementPeriodNs = 60 * 60 * NS_PER_SEC;

    /* Minimum period between two activation broadcasts in nanoseconds. */
    static const int64_t kMinActivationBroadcastPeriodNs = 10 * NS_PER_SEC;

    // Maximum age (30 days) that files on disk can exist in seconds.
    static const int kMaxAgeSecond = 60 * 60 * 24 * 30;

    // Maximum age (2 days) that local history files on disk can exist in seconds.
    static const int kMaxLocalHistoryAgeSecond = 60 * 60 * 24 * 2;

    // Maximum number of files (1000) that can be in stats directory on disk.
    static const int kMaxFileNumber = 1000;

    // Maximum size of all files that can be written to stats directory on disk.
    static const int kMaxFileSize = 50 * 1024 * 1024;

    // How long to try to clear puller cache from last time
    static const long kPullerCacheClearIntervalSec = 1;

    // Max time to do a pull.
    static const int64_t kPullMaxDelayNs = 30 * NS_PER_SEC;

    // Maximum number of pushed atoms statsd stats will track above kMaxPushedAtomId.
    static const int kMaxNonPlatformPushedAtoms = 600;

    // Maximum number of pushed atoms error statsd stats will track.
    static const int kMaxPushedAtomErrorStatsSize = 100;

    // Maximum number of socket loss stats to track.
    static const int kMaxSocketLossStatsSize = 50;

    // Maximum atom id value that we consider a platform pushed atom.
    // This should be updated once highest pushed atom id in atoms.proto approaches this value.
    static const int32_t kMaxPushedAtomId = 1500;

    // Atom id that is the start of the pulled atoms.
    static const int32_t kPullAtomStartTag = 10000;

    // Atom id that is the start of vendor atoms.
    static const int32_t kVendorAtomStartTag = 100000;

    // Vendor pulled atom start id.
    static const int32_t kVendorPulledAtomStartTag = 150000;

    // Beginning of range for timestamp truncation.
    static const int32_t kTimestampTruncationStartTag = 300000;

    // End of range for timestamp truncation.
    static const int32_t kTimestampTruncationEndTag = 304999;

    // Max accepted atom id.
    static const int32_t kMaxAtomTag = 200000;

    static const int32_t kMaxLoggedBucketDropEvents = 10;

    static const int32_t kNumBinsInSocketBatchReadHistogram = 30;
    static const int32_t kLargeBatchReadThreshold = 1000;
    static const int32_t kMaxLargeBatchReadSize = 20;
    static const int32_t kMaxLargeBatchReadAtomThreshold = 50;

    static const int32_t kMaxLoggingRateStatsToReport = 50;

    /**
     * Report a new config has been received and report the static stats about the config.
     *
     * The static stats include: the count of metrics, conditions, matchers, and alerts.
     * If the config is not valid, this config stats will be put into icebox immediately.
     */
    void noteConfigReceived(const ConfigKey& key, int metricsCount, int conditionsCount,
                            int matchersCount, int alertCount,
                            const std::list<std::pair<const int64_t, const int32_t>>& annotations,
                            const std::optional<InvalidConfigReason>& reason);
    /**
     * Report a config has been removed.
     */
    void noteConfigRemoved(const ConfigKey& key);
    /**
     * Report a config has been reset when ttl expires.
     */
    void noteConfigReset(const ConfigKey& key);

    /**
     * Report a broadcast has been sent to a config owner to collect the data.
     */
    void noteBroadcastSent(const ConfigKey& key);

    /**
     * Report that a config has become activated or deactivated.
     * This can be different from whether or not a broadcast is sent if the
     * guardrail prevented the broadcast from being sent.
     */
    void noteActiveStatusChanged(const ConfigKey& key, bool activate);

    /**
     * Report a config's metrics data has been dropped.
     */
    void noteDataDropped(const ConfigKey& key, const size_t totalBytes);

    /**
     * Report metrics data report has been sent.
     *
     * The report may be requested via StatsManager API, or through adb cmd.
     */
    void noteMetricsReportSent(const ConfigKey& key, const size_t numBytes,
                               const int32_t reportNumber);

    /**
     * Report failure in creating the device info metadata table for restricted configs.
     */
    void noteDeviceInfoTableCreationFailed(const ConfigKey& key);

    /**
     * Report db corruption for restricted configs.
     */
    void noteDbCorrupted(const ConfigKey& key);

    /**
     * Report db exceeded the size limit for restricted configs.
     */
    void noteDbSizeExceeded(const ConfigKey& key);

    /**
     * Report db size check with stat for restricted configs failed.
     */
    void noteDbStatFailed(const ConfigKey& key);

    /**
     * Report restricted config is invalid.
     */
    void noteDbConfigInvalid(const ConfigKey& key);

    /**
     * Report db is too old for restricted configs.
     */
    void noteDbTooOld(const ConfigKey& key);

    /**
     * Report db was deleted due to config removal.
     */
    void noteDbDeletionConfigRemoved(const ConfigKey& key);

    /**
     * Report db was deleted due to config update.
     */
    void noteDbDeletionConfigUpdated(const ConfigKey& key);

    /**
     * Reports that the promotion for ConfigMetadataProvider failed.
     */
    void noteConfigMetadataProviderPromotionFailed(const ConfigKey& key);

    /**
     * Report the size of output tuple of a condition.
     *
     * Note: only report when the condition has an output dimension, and the tuple
     * count > kDimensionKeySizeSoftLimit.
     *
     * [key]: The config key that this condition belongs to.
     * [id]: The id of the condition.
     * [size]: The output tuple size.
     */
    void noteConditionDimensionSize(const ConfigKey& key, int64_t id, int size);

    /**
     * Report the size of output tuple of a metric.
     *
     * Note: only report when the metric has an output dimension, and the tuple
     * count > kDimensionKeySizeSoftLimit.
     *
     * [key]: The config key that this metric belongs to.
     * [id]: The id of the metric.
     * [size]: The output tuple size.
     */
    void noteMetricDimensionSize(const ConfigKey& key, int64_t id, int size);

    /**
     * Report the max size of output tuple of dimension in condition across dimensions in what.
     *
     * Note: only report when the metric has an output dimension in condition, and the max tuple
     * count > kDimensionKeySizeSoftLimit.
     *
     * [key]: The config key that this metric belongs to.
     * [id]: The id of the metric.
     * [size]: The output tuple size.
     */
    void noteMetricDimensionInConditionSize(const ConfigKey& key, int64_t id, int size);

    /**
     * Report a matcher has been matched.
     *
     * [key]: The config key that this matcher belongs to.
     * [id]: The id of the matcher.
     */
    void noteMatcherMatched(const ConfigKey& key, int64_t id);

    /**
     * Report that an anomaly detection alert has been declared.
     *
     * [key]: The config key that this alert belongs to.
     * [id]: The id of the alert.
     */
    void noteAnomalyDeclared(const ConfigKey& key, int64_t id);

    /**
     * Report an atom event has been logged.
     */
    void noteAtomLogged(int atomId, int64_t eventTimestampNs, bool isSkipped);

    /**
     * Report that statsd modified the anomaly alarm registered with StatsCompanionService.
     */
    void noteRegisteredAnomalyAlarmChanged();

    /**
     * Report that statsd modified the periodic alarm registered with StatsCompanionService.
     */
    void noteRegisteredPeriodicAlarmChanged();

    /**
     * Records the number of delta entries that are being dropped from the uid map.
     */
    void noteUidMapDropped(int deltas);

    /**
     * Records that an app was deleted (from statsd's map).
     */
    void noteUidMapAppDeletionDropped();

    /**
     * Updates the number of changes currently stored in the uid map.
     */
    void setUidMapChanges(int changes);
    void setCurrentUidMapMemory(int bytes);

    /*
     * Updates minimum interval between pulls for an pulled atom.
     */
    void updateMinPullIntervalSec(int pullAtomId, long intervalSec);

    /*
     * Notes an atom is pulled.
     */
    void notePull(int pullAtomId);

    /*
     * Notes an atom is served from puller cache.
     */
    void notePullFromCache(int pullAtomId);

    /*
     * Notify data error for pulled atom.
     */
    void notePullDataError(int pullAtomId);

    /*
     * Records time for actual pulling, not including those served from cache and not including
     * statsd processing delays.
     */
    void notePullTime(int pullAtomId, int64_t pullTimeNs);

    /*
     * Records pull delay for a pulled atom, including those served from cache and including statsd
     * processing delays.
     */
    void notePullDelay(int pullAtomId, int64_t pullDelayNs);

    /*
     * Records pull exceeds timeout for the puller.
     */
    void notePullTimeout(int pullAtomId, int64_t pullUptimeMillis, int64_t pullElapsedMillis);

    /*
     * Records pull exceeds max delay for a metric.
     */
    void notePullExceedMaxDelay(int pullAtomId);

    /*
     * Records when system server restarts.
     */
    void noteSystemServerRestart(int32_t timeSec);

    /**
     * Records statsd skipped an event.
     */
    void noteLogLost(int32_t wallClockTimeSec, int32_t count, int32_t lastError,
                     int32_t lastAtomTag, int32_t uid, int32_t pid);

    /**
     * Records that the pull of an atom has failed. Eg, if the client indicated the pull failed, if
     * the pull timed out, or if the outgoing binder call failed.
     * This count will only increment if the puller was actually invoked.
     *
     * It does not include a pull not occurring due to not finding the appropriate
     * puller. These cases are covered in other counts.
     */
    void notePullFailed(int atomId);

    /**
     * Records that the pull of an atom has failed due to not having a uid provider.
     */
    void notePullUidProviderNotFound(int atomId);

    /**
     * Records that the pull of an atom has failed due not finding a puller registered by a
     * trusted uid.
     */
    void notePullerNotFound(int atomId);

    /**
     * Records that the pull has failed due to the outgoing binder call failing.
     */
    void notePullBinderCallFailed(int atomId);

    /**
     * A pull with no data occurred
     */
    void noteEmptyData(int atomId);

    /**
     * Records that a puller callback for the given atomId was registered or unregistered.
     *
     * @param registered True if the callback was registered, false if was unregistered.
     */
    void notePullerCallbackRegistrationChanged(int atomId, bool registered);

    /**
     * Hard limit was reached in the cardinality of an atom
     */
    void noteHardDimensionLimitReached(int64_t metricId);

    /**
     * A log event was too late, arrived in the wrong bucket and was skipped
     */
    void noteLateLogEventSkipped(int64_t metricId);

    /**
     * Buckets were skipped as time elapsed without any data for them
     */
    void noteSkippedForwardBuckets(int64_t metricId);

    /**
     * An unsupported value type was received
     */
    void noteBadValueType(int64_t metricId);

    /**
     * Buckets were dropped due to reclaim memory.
     */
    void noteBucketDropped(int64_t metricId);

    /**
     * A condition change was too late, arrived in the wrong bucket and was skipped
     */
    void noteConditionChangeInNextBucket(int64_t metricId);

    /**
     * A bucket has been tagged as invalid.
     */
    void noteInvalidatedBucket(int64_t metricId);

    /**
     * Tracks the total number of buckets (include skipped/invalid buckets).
     */
    void noteBucketCount(int64_t metricId);

    /**
     * For pulls at bucket boundaries, it represents the misalignment between the real timestamp and
     * the end of the bucket.
     */
    void noteBucketBoundaryDelayNs(int64_t metricId, int64_t timeDelayNs);

    /**
     * Number of buckets with unknown condition.
     */
    void noteBucketUnknownCondition(int64_t metricId);

    /* Reports one event id has been dropped due to queue overflow, and the oldest event timestamp
     * in the queue. There is an expectation that noteAtomLogged() is called for the same
     * atomId
     */
    void noteEventQueueOverflow(int64_t oldestEventTimestampNs, int32_t atomId);

    /* Notes queue max size seen so far and associated timestamp */
    void noteEventQueueSize(int32_t size, int64_t eventTimestampNs);

    /**
     * Reports that the activation broadcast guardrail was hit for this uid. Namely, the broadcast
     * should have been sent, but instead was skipped due to hitting the guardrail.
     */
    void noteActivationBroadcastGuardrailHit(const int uid);

    /**
     * Reports that an atom is erroneous or cannot be parsed successfully by
     * statsd. An atom tag of 0 indicates that the client did not supply the
     * atom id within the encoding.
     *
     * For pushed atoms only, this call should be preceded by a call to
     * noteAtomLogged.
     */
    void noteAtomError(int atomTag, bool pull = false);

    /**
     * Increases counter associated with a CounterType.
     */
    void noteIllegalState(CounterType error);

    /** Report query of restricted metric succeed **/
    void noteQueryRestrictedMetricSucceed(const int64_t configId, const string& configPackage,
                                          const std::optional<int32_t> configUid,
                                          const int32_t callingUid, int64_t queryLatencyNs);

    /** Report query of restricted metric failed **/
    void noteQueryRestrictedMetricFailed(const int64_t configId, const string& configPackage,
                                         const std::optional<int32_t> configUid,
                                         const int32_t callingUid, const InvalidQueryReason reason);

    /** Report query of restricted metric failed along with an error string **/
    void noteQueryRestrictedMetricFailed(const int64_t configId, const string& configPackage,
                                         const std::optional<int32_t> configUid,
                                         const int32_t callingUid, const InvalidQueryReason reason,
                                         const string& error);

    // Reports that a restricted metric fails to be inserted to database.
    void noteRestrictedMetricInsertError(const ConfigKey& configKey, int64_t metricId);

    // Reports that a restricted metric fails to create table in database.
    void noteRestrictedMetricTableCreationError(const ConfigKey& configKey, int64_t metricId);

    // Reports that a restricted metric fails to delete table in database.
    void noteRestrictedMetricTableDeletionError(const ConfigKey& configKey, int64_t metricId);

    // Reports the time it takes for a restricted metric to flush the data to the database.
    void noteRestrictedMetricFlushLatency(const ConfigKey& configKey, int64_t metricId,
                                          const int64_t flushLatencyNs);

    // Reports that a restricted metric had a category change.
    void noteRestrictedMetricCategoryChanged(const ConfigKey& configKey, int64_t metricId);

    // Reports the time is takes to flush a restricted config to the database.
    void noteRestrictedConfigFlushLatency(const ConfigKey& configKey,
                                          const int64_t totalFlushLatencyNs);

    // Reports the size of the internal sqlite db.
    void noteRestrictedConfigDbSize(const ConfigKey& configKey, int64_t elapsedTimeNs,
                                    const int64_t dbSize);

    /**
     * Records libstatssocket was not able to write into socket.
     */
    void noteAtomSocketLoss(const SocketLossInfo& lossInfo);

    /**
     * Report a new subscription has started and report the static stats about the subscription
     * config.
     *
     * The static stats include: the count of pushed atoms and pulled atoms.
     */
    void noteSubscriptionStarted(int subId, int32_t pushedAtomCount, int32_t pulledAtomCount);

    /**
     * Report an existing subscription has ended.
     */
    void noteSubscriptionEnded(int subId);

    /**
     * Report an existing subscription was flushed.
     */
    void noteSubscriptionFlushed(int subId);

    /**
     * Report an atom was pulled for a subscription.
     */
    void noteSubscriptionAtomPulled(int atomId);

    /**
     * Report subscriber pull thread wakeup.
     */
    void noteSubscriptionPullThreadWakeup();

    void noteBatchSocketRead(int32_t size, int64_t lastReadTimeNs, int64_t currReadTimeNs,
                             int64_t minAtomReadTimeNs, int64_t maxAtomReadTimeNs,
                             const std::unordered_map<int32_t, int32_t>& atomCounts);

    /**
     * Reset the historical stats. Including all stats in icebox, and the tracked stats about
     * metrics, matchers, and atoms. The active configs will be kept and StatsdStats will continue
     * to collect stats after reset() has been called.
     */
    void reset();

    /**
     * Output the stats in protobuf binary format to [buffer].
     *
     * [reset]: whether to clear the historical stats after the call.
     */
    void dumpStats(std::vector<uint8_t>* buffer, bool reset);

    /**
     * Output statsd stats in human readable format to [out] file descriptor.
     */
    void dumpStats(int outFd) const;

    /**
     * Returns true if dimension guardrail has been hit since boot for given metric.
     */
    bool hasHitDimensionGuardrail(int64_t metricId) const;

    /**
     * Return soft and hard atom key dimension size limits as an std::pair.
     */
    static std::pair<size_t, size_t> getAtomDimensionKeySizeLimits(int atomId,
                                                                   size_t defaultHardLimit);

    inline static int clampDimensionKeySizeLimit(int dimLimit) {
        return std::clamp(dimLimit, kDimensionKeySizeHardLimitMin, kDimensionKeySizeHardLimitMax);
    }

    /**
     * Return the unique identifier for the statsd stats report. This id is
     * reset on boot.
     */
    inline int32_t getStatsdStatsId() const {
        return mStatsdStatsId;
    }

    /**
     * Returns true if there is recorded event queue overflow
     */
    bool hasEventQueueOverflow() const;

    typedef std::unordered_map<int32_t, int32_t> QueueOverflowAtomsStatsMap;
    QueueOverflowAtomsStatsMap getQueueOverflowAtomsStats() const;

    /**
     * Returns true if there is recorded socket loss
     */
    bool hasSocketLoss() const;

    typedef struct PullTimeoutMetadata {
        int64_t pullTimeoutUptimeMillis;
        int64_t pullTimeoutElapsedMillis;
        PullTimeoutMetadata(int64_t uptimeMillis, int64_t elapsedMillis)
            : pullTimeoutUptimeMillis(uptimeMillis),
              pullTimeoutElapsedMillis(elapsedMillis) { /* do nothing */
        }
    } PullTimeoutMetadata;

    typedef struct {
        long totalPull = 0;
        long totalPullFromCache = 0;
        long minPullIntervalSec = LONG_MAX;
        int64_t avgPullTimeNs = 0;
        int64_t maxPullTimeNs = 0;
        long numPullTime = 0;
        int64_t avgPullDelayNs = 0;
        int64_t maxPullDelayNs = 0;
        long numPullDelay = 0;
        long dataError = 0;
        long pullTimeout = 0;
        long pullExceedMaxDelay = 0;
        long pullFailed = 0;
        long pullUidProviderNotFound = 0;
        long pullerNotFound = 0;
        long emptyData = 0;
        long registeredCount = 0;
        long unregisteredCount = 0;
        int32_t atomErrorCount = 0;
        long binderCallFailCount = 0;
        std::list<PullTimeoutMetadata> pullTimeoutMetadata;
        int32_t subscriptionPullCount = 0;
    } PulledAtomStats;

    typedef struct {
        long hardDimensionLimitReached = 0;
        long lateLogEventSkipped = 0;
        long skippedForwardBuckets = 0;
        long badValueType = 0;
        long conditionChangeInNextBucket = 0;
        long invalidatedBucket = 0;
        long bucketDropped = 0;
        int64_t minBucketBoundaryDelayNs = 0;
        int64_t maxBucketBoundaryDelayNs = 0;
        long bucketUnknownCondition = 0;
        long bucketCount = 0;
    } AtomMetricStats;

private:
    StatsdStats();

    mutable std::mutex mLock;

    int32_t mStartTimeSec;

    // Random id set using rand() during the initialization. Used to uniquely
    // identify a session. This is more reliable than mStartTimeSec due to the
    // unreliable nature of wall clock times.
    const int32_t mStatsdStatsId;

    // Track the number of dropped entries used by the uid map.
    UidMapStats mUidMapStats;

    // The stats about the configs that are still in use.
    // The map size is capped by kMaxConfigCount.
    std::map<const ConfigKey, std::shared_ptr<ConfigStats>> mConfigStats;

    // Stores the stats for the configs that are no longer in use.
    // The size of the vector is capped by kMaxIceBoxSize.
    std::list<std::shared_ptr<ConfigStats>> mIceBox;

    // Stores the number of times a pushed atom is logged and skipped (if skipped).
    // The size of the vector is the largest pushed atom id in atoms.proto + 1. Atoms
    // out of that range will be put in mNonPlatformPushedAtomStats.
    // This is a vector, not a map because it will be accessed A LOT -- for each stats log.
    struct PushedAtomStats {
        int logCount = 0;
        int skipCount = 0;
    };

    std::vector<PushedAtomStats> mPushedAtomStats;

    // Stores the number of times a pushed atom is logged and skipped for atom ids above
    // kMaxPushedAtomId. The max size of the map is kMaxNonPlatformPushedAtoms.
    std::unordered_map<int, PushedAtomStats> mNonPlatformPushedAtomStats;

    // Stores the number of times a pushed atom is dropped due to queue overflow event.
    // We do not expect it will happen too often so the map is preferable vs pre-allocated vector
    // The max size of the map is kMaxPushedAtomId + kMaxNonPlatformPushedAtoms.
    QueueOverflowAtomsStatsMap mPushedAtomDropsStats;

    // Maps PullAtomId to its stats. The size is capped by the puller atom counts.
    std::map<int, PulledAtomStats> mPulledAtomStats;

    // Tracks counter associated with CounterType to represent errors. Max capacity == CounterType
    std::map<CounterType, int32_t> mErrorStats;

    // Stores the number of times a pushed atom was logged erroneously. The
    // corresponding counts for pulled atoms are stored in PulledAtomStats.
    // The max size of this map is kMaxPushedAtomErrorStatsSize.
    std::map<int, int> mPushedAtomErrorStats;

    // Stores the number of times a pushed atom was lost due to socket error.
    // Represents counter per uid per tag per error with indication when the loss event was observed
    // first & last time.
    struct SocketLossStats {
        SocketLossStats(int32_t uid, int64_t firstLossTsNanos, int64_t lastLossTsNanos)
            : mUid(uid), mFirstLossTsNanos(firstLossTsNanos), mLastLossTsNanos(lastLossTsNanos) {
        }

        int32_t mUid;
        int64_t mFirstLossTsNanos;
        int64_t mLastLossTsNanos;
        // atom loss count per error, atom id
        struct AtomLossInfo {
            AtomLossInfo(int32_t atomId, int32_t error, int32_t count)
                : mAtomId(atomId), mError(error), mCount(count) {
            }
            int mAtomId;
            int mError;
            int mCount;
        };
        std::vector<AtomLossInfo> mLossCountPerErrorAtomId;
    };
    // The max size of this list is kMaxSocketLossStatsSize.
    std::list<SocketLossStats> mSocketLossStats;

    // Stores the number of times a pushed atom loss info was dropped from the stats
    // on libstatssocket side due to guardrail hit.
    // Represents counter per uid.
    // The max size of this map is kMaxSocketLossStatsSize.
    std::map<int32_t, int32_t> mSocketLossStatsOverflowCounters;

    // Maps metric ID to its stats. The size is capped by the number of metrics.
    std::map<int64_t, AtomMetricStats> mAtomMetricStats;

    // Maps uids to times when the activation changed broadcast not sent due to hitting the
    // guardrail. The size is capped by the number of configs, and up to 20 times per uid.
    std::map<int, std::list<int32_t>> mActivationBroadcastGuardrailStats;

    struct LogLossStats {
        LogLossStats(int32_t sec, int32_t count, int32_t error, int32_t tag, int32_t uid,
                     int32_t pid)
            : mWallClockSec(sec),
              mCount(count),
              mLastError(error),
              mLastTag(tag),
              mUid(uid),
              mPid(pid) {
        }
        int32_t mWallClockSec;
        int32_t mCount;
        // error code defined in linux/errno.h
        int32_t mLastError;
        int32_t mLastTag;
        int32_t mUid;
        int32_t mPid;
    };

    // Max of {(now - oldestEventTimestamp) when overflow happens}.
    // This number is helpful to understand how SLOW statsd can be.
    int64_t mMaxQueueHistoryNs = 0;

    // Min of {(now - oldestEventTimestamp) when overflow happens}.
    // This number is helpful to understand how FAST the events floods to statsd.
    int64_t mMinQueueHistoryNs = std::numeric_limits<int64_t>::max();

    // Total number of events that are lost due to queue overflow.
    int32_t mOverflowCount = 0;

    // Max number of events stored into the queue seen so far.
    int32_t mEventQueueMaxSizeObserved = 0;

    // Event timestamp for associated max size hit.
    int64_t mEventQueueMaxSizeObservedElapsedNanos = 0;

    // Timestamps when we detect log loss, and the number of logs lost.
    std::list<LogLossStats> mLogLossStats;

    LoggingRate mLoggingRateStats;

    std::list<int32_t> mSystemServerRestartSec;

    std::vector<int64_t> mSocketBatchReadHistogram;

    // Stores stats about large socket batch reads
    struct LargeBatchSocketReadStats {
        LargeBatchSocketReadStats(int32_t size, int64_t lastReadTimeNs, int64_t currReadTimeNs,
                                  int64_t minAtomReadTimeNs, int64_t maxAtomReadTimeNs,
                                  const std::unordered_map<int32_t, int32_t>& atomCounts)
            : mSize(size),
              mLastReadTimeNs(lastReadTimeNs),
              mCurrReadTimeNs(currReadTimeNs),
              mMinAtomReadTimeNs(minAtomReadTimeNs),
              mMaxAtomReadTimeNs(maxAtomReadTimeNs),
              mCommonAtomCounts(atomCounts) {
        }

        int32_t mSize;
        // The elapsed time of the previous and current read times.
        int64_t mLastReadTimeNs;
        int64_t mCurrReadTimeNs;
        // The min and max times of the LogEvents processed in the batch
        int64_t mMinAtomReadTimeNs;
        int64_t mMaxAtomReadTimeNs;
        // Map of atom id to count for atoms logged more than kMaxLargeBatchReadAtomThreshold times.
        std::unordered_map<int32_t, int32_t> mCommonAtomCounts;
    };
    // The max size of this list is kMaxSocketLossStatsSize.
    std::list<LargeBatchSocketReadStats> mLargeBatchSocketReadStats;

    struct RestrictedMetricQueryStats {
        RestrictedMetricQueryStats(int32_t callingUid, int64_t configId,
                                   const string& configPackage, std::optional<int32_t> configUid,
                                   int64_t queryTimeNs,
                                   std::optional<InvalidQueryReason> invalidQueryReason,
                                   const string& error, std::optional<int64_t> queryLatencyNs)
            : mCallingUid(callingUid),
              mConfigId(configId),
              mConfigPackage(configPackage),
              mConfigUid(configUid),
              mQueryWallTimeNs(queryTimeNs),
              mInvalidQueryReason(invalidQueryReason),
              mError(error),
              mQueryLatencyNs(queryLatencyNs) {
            mHasError = invalidQueryReason.has_value();
        }
        int32_t mCallingUid;
        int64_t mConfigId;
        string mConfigPackage;
        std::optional<int32_t> mConfigUid;
        int64_t mQueryWallTimeNs;
        std::optional<InvalidQueryReason> mInvalidQueryReason;
        bool mHasError;
        string mError;
        std::optional<int64_t> mQueryLatencyNs;
    };
    std::list<RestrictedMetricQueryStats> mRestrictedMetricQueryStats;

    void noteQueryRestrictedMetricFailedLocked(const int64_t configId, const string& configPackage,
                                               const std::optional<int32_t> configUid,
                                               const int32_t callingUid,
                                               const InvalidQueryReason reason,
                                               const string& error);

    int32_t mSubscriptionPullThreadWakeupCount = 0;

    // Maps Subscription ID to the corresponding SubscriptionStats struct object.
    // Size of this map is capped by ShellSubscriber::kMaxSubscriptions.
    std::map<int32_t, SubscriptionStats> mSubscriptionStats;

    // Stores the number of times statsd modified the anomaly alarm registered with
    // StatsCompanionService.
    int mAnomalyAlarmRegisteredStats = 0;

    // Stores the number of times statsd registers the periodic alarm changes
    int mPeriodicAlarmRegisteredStats = 0;

    void noteConfigResetInternalLocked(const ConfigKey& key);

    void noteConfigRemovedInternalLocked(const ConfigKey& key);

    void resetInternalLocked();

    void noteAtomLoggedLocked(int atomId, int64_t eventTimestampNs, bool isSkipped);

    void noteAtomDroppedLocked(int atomId);

    void noteDataDropped(const ConfigKey& key, const size_t totalBytes, int32_t timeSec);

    void noteMetricsReportSent(const ConfigKey& key, const size_t numBytes, int32_t timeSec,
                               const int32_t reportNumber);

    void noteBroadcastSent(const ConfigKey& key, int32_t timeSec);

    void noteActiveStatusChanged(const ConfigKey& key, bool activate, int32_t timeSec);

    void noteActivationBroadcastGuardrailHit(const int uid, int32_t timeSec);

    void addToIceBoxLocked(std::shared_ptr<ConfigStats>& stats);

    int getPushedAtomErrorsLocked(int atomId) const;

    int getPushedAtomDropsLocked(int atomId) const;

    int getLoggingRateLocked(int atomId) const;

    bool hasRestrictedConfigErrors(const std::shared_ptr<ConfigStats>& configStats) const;

    /**
     * Get a reference to AtomMetricStats for a metric. If none exists, create it. The reference
     * will live as long as `this`.
     */
    StatsdStats::AtomMetricStats& getAtomMetricStats(int64_t metricId);

    FRIEND_TEST(LogEventQueue_test, TestQueueMaxSize);
    FRIEND_TEST(SocketParseMessageTest, TestProcessMessage);
    FRIEND_TEST(StatsLogProcessorTest, InvalidConfigRemoved);
    FRIEND_TEST(StatsPullerManagerTest, TestOnAlarmFiredNoPullerForUidNotesPullerNotFound);
    FRIEND_TEST(StatsPullerManagerTest, TestOnAlarmFiredNoUidProviderUpdatesNextPullTime);
    FRIEND_TEST(StatsPullerManagerTest, TestOnAlarmFiredUidsNotRegisteredInPullAtomCallback);
    FRIEND_TEST(StatsdStatsTest, TestActivationBroadcastGuardrailHit);
    FRIEND_TEST(StatsdStatsTest, TestAnomalyMonitor);
    FRIEND_TEST(StatsdStatsTest, TestAtomDroppedStats);
    FRIEND_TEST(StatsdStatsTest, TestAtomErrorStats);
    FRIEND_TEST(StatsdStatsTest, TestAtomLog);
    FRIEND_TEST(StatsdStatsTest, TestAtomLoggedAndDroppedAndSkippedStats);
    FRIEND_TEST(StatsdStatsTest, TestAtomLoggedAndDroppedStats);
    FRIEND_TEST(StatsdStatsTest, TestAtomMetricsStats);
    FRIEND_TEST(StatsdStatsTest, TestAtomSkippedStats);
    FRIEND_TEST(StatsdStatsTest, TestConfigMetadataProviderPromotionFailed);
    FRIEND_TEST(StatsdStatsTest, TestConfigRemove);
    FRIEND_TEST(StatsdStatsTest, TestHasHitDimensionGuardrail);
    FRIEND_TEST(StatsdStatsTest, TestInvalidConfigAdd);
    FRIEND_TEST(StatsdStatsTest, TestInvalidConfigMissingMetricId);
    FRIEND_TEST(StatsdStatsTest, TestInvalidConfigOnlyMetricId);
    FRIEND_TEST(StatsdStatsTest, TestNonPlatformAtomLog);
    FRIEND_TEST(StatsdStatsTest, TestPullAtomStats);
    FRIEND_TEST(StatsdStatsTest, TestQueueStats);
    FRIEND_TEST(StatsdStatsTest, TestRestrictedMetricsQueryStats);
    FRIEND_TEST(StatsdStatsTest, TestRestrictedMetricsStats);
    FRIEND_TEST(StatsdStatsTest, TestShardOffsetProvider);
    FRIEND_TEST(StatsdStatsTest, TestSocketLossStats);
    FRIEND_TEST(StatsdStatsTest, TestSocketLossStatsOverflowCounter);
    FRIEND_TEST(StatsdStatsTest, TestSubStats);
    FRIEND_TEST(StatsdStatsTest, TestSubscriptionAtomPulled);
    FRIEND_TEST(StatsdStatsTest, TestSubscriptionEnded);
    FRIEND_TEST(StatsdStatsTest, TestSubscriptionFlushed);
    FRIEND_TEST(StatsdStatsTest, TestSubscriptionPullThreadWakeup);
    FRIEND_TEST(StatsdStatsTest, TestSubscriptionStarted);
    FRIEND_TEST(StatsdStatsTest, TestSubscriptionStartedMaxActiveSubscriptions);
    FRIEND_TEST(StatsdStatsTest, TestSubscriptionStartedRemoveFinishedSubscription);
    FRIEND_TEST(StatsdStatsTest, TestSystemServerCrash);
    FRIEND_TEST(StatsdStatsTest, TestTimestampThreshold);
    FRIEND_TEST(StatsdStatsTest, TestValidConfigAdd);
    FRIEND_TEST(StatsdStatsTest, TestSocketBatchReadStats);
    FRIEND_TEST(StatsdStatsTest, TestErrorStatsReport);
    FRIEND_TEST(StatsdStatsTest, TestErrorStatsReportReset);
    FRIEND_TEST(StatsdStatsTest, TestLoggingRateReport);
    FRIEND_TEST(StatsdStatsTest, TestLoggingRateReportOnlyTopN);
    FRIEND_TEST(StatsdStatsTest, TestLoggingRateReportReset);
};

InvalidConfigReason createInvalidConfigReasonWithMatcher(const InvalidConfigReasonEnum reason,
                                                         const int64_t matcherId);

InvalidConfigReason createInvalidConfigReasonWithMatcher(const InvalidConfigReasonEnum reason,
                                                         const int64_t metricId,
                                                         const int64_t matcherId);

InvalidConfigReason createInvalidConfigReasonWithPredicate(const InvalidConfigReasonEnum reason,
                                                           const int64_t conditionId);

InvalidConfigReason createInvalidConfigReasonWithPredicate(const InvalidConfigReasonEnum reason,
                                                           const int64_t metricId,
                                                           const int64_t conditionId);

InvalidConfigReason createInvalidConfigReasonWithState(const InvalidConfigReasonEnum reason,
                                                       const int64_t metricId,
                                                       const int64_t stateId);

InvalidConfigReason createInvalidConfigReasonWithAlert(const InvalidConfigReasonEnum reason,
                                                       const int64_t alertId);

InvalidConfigReason createInvalidConfigReasonWithAlert(const InvalidConfigReasonEnum reason,
                                                       const int64_t metricId,
                                                       const int64_t alertId);

InvalidConfigReason createInvalidConfigReasonWithAlarm(const InvalidConfigReasonEnum reason,
                                                       const int64_t alarmId);

InvalidConfigReason createInvalidConfigReasonWithSubscription(const InvalidConfigReasonEnum reason,
                                                              const int64_t subscriptionId);

InvalidConfigReason createInvalidConfigReasonWithSubscriptionAndAlarm(
        const InvalidConfigReasonEnum reason, int64_t subscriptionId, int64_t alarmId);

InvalidConfigReason createInvalidConfigReasonWithSubscriptionAndAlert(
        const InvalidConfigReasonEnum reason, int64_t subscriptionId, int64_t alertId);

}  // namespace statsd
}  // namespace os
}  // namespace android