1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "net/url_request/url_request_throttler_manager.h"
6
7 #include "base/logging.h"
8 #include "base/metrics/field_trial.h"
9 #include "base/metrics/histogram.h"
10 #include "base/strings/string_util.h"
11 #include "net/base/net_log.h"
12 #include "net/base/net_util.h"
13
14 namespace net {
15
16 const unsigned int URLRequestThrottlerManager::kMaximumNumberOfEntries = 1500;
17 const unsigned int URLRequestThrottlerManager::kRequestsBetweenCollecting = 200;
18
URLRequestThrottlerManager()19 URLRequestThrottlerManager::URLRequestThrottlerManager()
20 : requests_since_last_gc_(0),
21 enable_thread_checks_(false),
22 logged_for_localhost_disabled_(false),
23 registered_from_thread_(base::kInvalidThreadId) {
24 url_id_replacements_.ClearPassword();
25 url_id_replacements_.ClearUsername();
26 url_id_replacements_.ClearQuery();
27 url_id_replacements_.ClearRef();
28
29 NetworkChangeNotifier::AddIPAddressObserver(this);
30 NetworkChangeNotifier::AddConnectionTypeObserver(this);
31 }
32
~URLRequestThrottlerManager()33 URLRequestThrottlerManager::~URLRequestThrottlerManager() {
34 NetworkChangeNotifier::RemoveIPAddressObserver(this);
35 NetworkChangeNotifier::RemoveConnectionTypeObserver(this);
36
37 // Since the manager object might conceivably go away before the
38 // entries, detach the entries' back-pointer to the manager.
39 UrlEntryMap::iterator i = url_entries_.begin();
40 while (i != url_entries_.end()) {
41 if (i->second.get() != NULL) {
42 i->second->DetachManager();
43 }
44 ++i;
45 }
46
47 // Delete all entries.
48 url_entries_.clear();
49 }
50
51 scoped_refptr<URLRequestThrottlerEntryInterface>
RegisterRequestUrl(const GURL & url)52 URLRequestThrottlerManager::RegisterRequestUrl(const GURL &url) {
53 DCHECK(!enable_thread_checks_ || CalledOnValidThread());
54
55 // Normalize the url.
56 std::string url_id = GetIdFromUrl(url);
57
58 // Periodically garbage collect old entries.
59 GarbageCollectEntriesIfNecessary();
60
61 // Find the entry in the map or create a new NULL entry.
62 scoped_refptr<URLRequestThrottlerEntry>& entry = url_entries_[url_id];
63
64 // If the entry exists but could be garbage collected at this point, we
65 // start with a fresh entry so that we possibly back off a bit less
66 // aggressively (i.e. this resets the error count when the entry's URL
67 // hasn't been requested in long enough).
68 if (entry.get() && entry->IsEntryOutdated()) {
69 entry = NULL;
70 }
71
72 // Create the entry if needed.
73 if (entry.get() == NULL) {
74 entry = new URLRequestThrottlerEntry(this, url_id);
75
76 // We only disable back-off throttling on an entry that we have
77 // just constructed. This is to allow unit tests to explicitly override
78 // the entry for localhost URLs. Given that we do not attempt to
79 // disable throttling for entries already handed out (see comment
80 // in AddToOptOutList), this is not a problem.
81 std::string host = url.host();
82 if (opt_out_hosts_.find(host) != opt_out_hosts_.end() ||
83 IsLocalhost(host)) {
84 if (!logged_for_localhost_disabled_ && IsLocalhost(host)) {
85 logged_for_localhost_disabled_ = true;
86 net_log_.AddEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST,
87 NetLog::StringCallback("host", &host));
88 }
89
90 // TODO(joi): Once sliding window is separate from back-off throttling,
91 // we can simply return a dummy implementation of
92 // URLRequestThrottlerEntryInterface here that never blocks anything (and
93 // not keep entries in url_entries_ for opted-out sites).
94 entry->DisableBackoffThrottling();
95 }
96 }
97
98 return entry;
99 }
100
AddToOptOutList(const std::string & host)101 void URLRequestThrottlerManager::AddToOptOutList(const std::string& host) {
102 // There is an edge case here that we are not handling, to keep things
103 // simple. If a host starts adding the opt-out header to its responses
104 // after there are already one or more entries in url_entries_ for that
105 // host, the pre-existing entries may still perform back-off throttling.
106 // In practice, this would almost never occur.
107 if (opt_out_hosts_.find(host) == opt_out_hosts_.end()) {
108 UMA_HISTOGRAM_COUNTS("Throttling.SiteOptedOut", 1);
109
110 net_log_.EndEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST,
111 NetLog::StringCallback("host", &host));
112 opt_out_hosts_.insert(host);
113 }
114 }
115
OverrideEntryForTests(const GURL & url,URLRequestThrottlerEntry * entry)116 void URLRequestThrottlerManager::OverrideEntryForTests(
117 const GURL& url,
118 URLRequestThrottlerEntry* entry) {
119 // Normalize the url.
120 std::string url_id = GetIdFromUrl(url);
121
122 // Periodically garbage collect old entries.
123 GarbageCollectEntriesIfNecessary();
124
125 url_entries_[url_id] = entry;
126 }
127
EraseEntryForTests(const GURL & url)128 void URLRequestThrottlerManager::EraseEntryForTests(const GURL& url) {
129 // Normalize the url.
130 std::string url_id = GetIdFromUrl(url);
131 url_entries_.erase(url_id);
132 }
133
set_enable_thread_checks(bool enable)134 void URLRequestThrottlerManager::set_enable_thread_checks(bool enable) {
135 enable_thread_checks_ = enable;
136 }
137
enable_thread_checks() const138 bool URLRequestThrottlerManager::enable_thread_checks() const {
139 return enable_thread_checks_;
140 }
141
set_net_log(NetLog * net_log)142 void URLRequestThrottlerManager::set_net_log(NetLog* net_log) {
143 DCHECK(net_log);
144 net_log_ = BoundNetLog::Make(net_log,
145 NetLog::SOURCE_EXPONENTIAL_BACKOFF_THROTTLING);
146 }
147
net_log() const148 NetLog* URLRequestThrottlerManager::net_log() const {
149 return net_log_.net_log();
150 }
151
OnIPAddressChanged()152 void URLRequestThrottlerManager::OnIPAddressChanged() {
153 OnNetworkChange();
154 }
155
OnConnectionTypeChanged(NetworkChangeNotifier::ConnectionType type)156 void URLRequestThrottlerManager::OnConnectionTypeChanged(
157 NetworkChangeNotifier::ConnectionType type) {
158 OnNetworkChange();
159 }
160
GetIdFromUrl(const GURL & url) const161 std::string URLRequestThrottlerManager::GetIdFromUrl(const GURL& url) const {
162 if (!url.is_valid())
163 return url.possibly_invalid_spec();
164
165 GURL id = url.ReplaceComponents(url_id_replacements_);
166 return StringToLowerASCII(id.spec()).c_str();
167 }
168
GarbageCollectEntriesIfNecessary()169 void URLRequestThrottlerManager::GarbageCollectEntriesIfNecessary() {
170 requests_since_last_gc_++;
171 if (requests_since_last_gc_ < kRequestsBetweenCollecting)
172 return;
173 requests_since_last_gc_ = 0;
174
175 GarbageCollectEntries();
176 }
177
GarbageCollectEntries()178 void URLRequestThrottlerManager::GarbageCollectEntries() {
179 UrlEntryMap::iterator i = url_entries_.begin();
180 while (i != url_entries_.end()) {
181 if ((i->second)->IsEntryOutdated()) {
182 url_entries_.erase(i++);
183 } else {
184 ++i;
185 }
186 }
187
188 // In case something broke we want to make sure not to grow indefinitely.
189 while (url_entries_.size() > kMaximumNumberOfEntries) {
190 url_entries_.erase(url_entries_.begin());
191 }
192 }
193
OnNetworkChange()194 void URLRequestThrottlerManager::OnNetworkChange() {
195 // Remove all entries. Any entries that in-flight requests have a reference
196 // to will live until those requests end, and these entries may be
197 // inconsistent with new entries for the same URLs, but since what we
198 // want is a clean slate for the new connection type, this is OK.
199 url_entries_.clear();
200 requests_since_last_gc_ = 0;
201 }
202
203 } // namespace net
204