1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "net/disk_cache/backend_impl.h"
6
7 #include "base/bind.h"
8 #include "base/bind_helpers.h"
9 #include "base/file_util.h"
10 #include "base/files/file_path.h"
11 #include "base/hash.h"
12 #include "base/message_loop/message_loop.h"
13 #include "base/metrics/field_trial.h"
14 #include "base/metrics/histogram.h"
15 #include "base/metrics/stats_counters.h"
16 #include "base/rand_util.h"
17 #include "base/strings/string_util.h"
18 #include "base/strings/stringprintf.h"
19 #include "base/sys_info.h"
20 #include "base/threading/thread_restrictions.h"
21 #include "base/time/time.h"
22 #include "base/timer/timer.h"
23 #include "net/base/net_errors.h"
24 #include "net/disk_cache/cache_util.h"
25 #include "net/disk_cache/entry_impl.h"
26 #include "net/disk_cache/errors.h"
27 #include "net/disk_cache/experiments.h"
28 #include "net/disk_cache/file.h"
29
30 // This has to be defined before including histogram_macros.h from this file.
31 #define NET_DISK_CACHE_BACKEND_IMPL_CC_
32 #include "net/disk_cache/histogram_macros.h"
33
34 using base::Time;
35 using base::TimeDelta;
36 using base::TimeTicks;
37
38 namespace {
39
40 const char* kIndexName = "index";
41
42 // Seems like ~240 MB correspond to less than 50k entries for 99% of the people.
43 // Note that the actual target is to keep the index table load factor under 55%
44 // for most users.
45 const int k64kEntriesStore = 240 * 1000 * 1000;
46 const int kBaseTableLen = 64 * 1024;
47 const int kDefaultCacheSize = 80 * 1024 * 1024;
48
49 // Avoid trimming the cache for the first 5 minutes (10 timer ticks).
50 const int kTrimDelay = 10;
51
DesiredIndexTableLen(int32 storage_size)52 int DesiredIndexTableLen(int32 storage_size) {
53 if (storage_size <= k64kEntriesStore)
54 return kBaseTableLen;
55 if (storage_size <= k64kEntriesStore * 2)
56 return kBaseTableLen * 2;
57 if (storage_size <= k64kEntriesStore * 4)
58 return kBaseTableLen * 4;
59 if (storage_size <= k64kEntriesStore * 8)
60 return kBaseTableLen * 8;
61
62 // The biggest storage_size for int32 requires a 4 MB table.
63 return kBaseTableLen * 16;
64 }
65
MaxStorageSizeForTable(int table_len)66 int MaxStorageSizeForTable(int table_len) {
67 return table_len * (k64kEntriesStore / kBaseTableLen);
68 }
69
GetIndexSize(int table_len)70 size_t GetIndexSize(int table_len) {
71 size_t table_size = sizeof(disk_cache::CacheAddr) * table_len;
72 return sizeof(disk_cache::IndexHeader) + table_size;
73 }
74
75 // ------------------------------------------------------------------------
76
77 // Sets group for the current experiment. Returns false if the files should be
78 // discarded.
InitExperiment(disk_cache::IndexHeader * header,bool cache_created)79 bool InitExperiment(disk_cache::IndexHeader* header, bool cache_created) {
80 if (header->experiment == disk_cache::EXPERIMENT_OLD_FILE1 ||
81 header->experiment == disk_cache::EXPERIMENT_OLD_FILE2) {
82 // Discard current cache.
83 return false;
84 }
85
86 if (base::FieldTrialList::FindFullName("SimpleCacheTrial") ==
87 "ExperimentControl") {
88 if (cache_created) {
89 header->experiment = disk_cache::EXPERIMENT_SIMPLE_CONTROL;
90 return true;
91 } else if (header->experiment != disk_cache::EXPERIMENT_SIMPLE_CONTROL) {
92 return false;
93 }
94 }
95
96 header->experiment = disk_cache::NO_EXPERIMENT;
97 return true;
98 }
99
100 } // namespace
101
102 // ------------------------------------------------------------------------
103
104 namespace disk_cache {
105
BackendImpl(const base::FilePath & path,base::MessageLoopProxy * cache_thread,net::NetLog * net_log)106 BackendImpl::BackendImpl(const base::FilePath& path,
107 base::MessageLoopProxy* cache_thread,
108 net::NetLog* net_log)
109 : background_queue_(this, cache_thread),
110 path_(path),
111 block_files_(path),
112 mask_(0),
113 max_size_(0),
114 up_ticks_(0),
115 cache_type_(net::DISK_CACHE),
116 uma_report_(0),
117 user_flags_(0),
118 init_(false),
119 restarted_(false),
120 unit_test_(false),
121 read_only_(false),
122 disabled_(false),
123 new_eviction_(false),
124 first_timer_(true),
125 user_load_(false),
126 net_log_(net_log),
127 done_(true, false),
128 ptr_factory_(this) {
129 }
130
SyncInit()131 int BackendImpl::SyncInit() {
132 #if defined(NET_BUILD_STRESS_CACHE)
133 // Start evictions right away.
134 up_ticks_ = kTrimDelay * 2;
135 #endif
136 DCHECK(!init_);
137 if (init_)
138 return net::ERR_FAILED;
139
140 bool create_files = false;
141 if (!InitBackingStore(&create_files)) {
142 ReportError(ERR_STORAGE_ERROR);
143 return net::ERR_FAILED;
144 }
145
146 num_refs_ = num_pending_io_ = max_refs_ = 0;
147 entry_count_ = byte_count_ = 0;
148
149 if (!restarted_) {
150 buffer_bytes_ = 0;
151 trace_object_ = TraceObject::GetTraceObject();
152 // Create a recurrent timer of 30 secs.
153 int timer_delay = unit_test_ ? 1000 : 30000;
154 timer_.reset(new base::RepeatingTimer<BackendImpl>());
155 timer_->Start(FROM_HERE, TimeDelta::FromMilliseconds(timer_delay), this,
156 &BackendImpl::OnStatsTimer);
157 }
158
159 init_ = true;
160 Trace("Init");
161
162 if (data_->header.experiment != NO_EXPERIMENT &&
163 cache_type_ != net::DISK_CACHE) {
164 // No experiment for other caches.
165 return net::ERR_FAILED;
166 }
167
168 if (!(user_flags_ & kNoRandom)) {
169 // The unit test controls directly what to test.
170 new_eviction_ = (cache_type_ == net::DISK_CACHE);
171 }
172
173 if (!CheckIndex()) {
174 ReportError(ERR_INIT_FAILED);
175 return net::ERR_FAILED;
176 }
177
178 if (!restarted_ && (create_files || !data_->header.num_entries))
179 ReportError(ERR_CACHE_CREATED);
180
181 if (!(user_flags_ & kNoRandom) && cache_type_ == net::DISK_CACHE &&
182 !InitExperiment(&data_->header, create_files)) {
183 return net::ERR_FAILED;
184 }
185
186 // We don't care if the value overflows. The only thing we care about is that
187 // the id cannot be zero, because that value is used as "not dirty".
188 // Increasing the value once per second gives us many years before we start
189 // having collisions.
190 data_->header.this_id++;
191 if (!data_->header.this_id)
192 data_->header.this_id++;
193
194 bool previous_crash = (data_->header.crash != 0);
195 data_->header.crash = 1;
196
197 if (!block_files_.Init(create_files))
198 return net::ERR_FAILED;
199
200 // We want to minimize the changes to cache for an AppCache.
201 if (cache_type() == net::APP_CACHE) {
202 DCHECK(!new_eviction_);
203 read_only_ = true;
204 } else if (cache_type() == net::SHADER_CACHE) {
205 DCHECK(!new_eviction_);
206 }
207
208 eviction_.Init(this);
209
210 // stats_ and rankings_ may end up calling back to us so we better be enabled.
211 disabled_ = false;
212 if (!InitStats())
213 return net::ERR_FAILED;
214
215 disabled_ = !rankings_.Init(this, new_eviction_);
216
217 #if defined(STRESS_CACHE_EXTENDED_VALIDATION)
218 trace_object_->EnableTracing(false);
219 int sc = SelfCheck();
220 if (sc < 0 && sc != ERR_NUM_ENTRIES_MISMATCH)
221 NOTREACHED();
222 trace_object_->EnableTracing(true);
223 #endif
224
225 if (previous_crash) {
226 ReportError(ERR_PREVIOUS_CRASH);
227 } else if (!restarted_) {
228 ReportError(ERR_NO_ERROR);
229 }
230
231 FlushIndex();
232
233 return disabled_ ? net::ERR_FAILED : net::OK;
234 }
235
PrepareForRestart()236 void BackendImpl::PrepareForRestart() {
237 // Reset the mask_ if it was not given by the user.
238 if (!(user_flags_ & kMask))
239 mask_ = 0;
240
241 if (!(user_flags_ & kNewEviction))
242 new_eviction_ = false;
243
244 disabled_ = true;
245 data_->header.crash = 0;
246 index_->Flush();
247 index_ = NULL;
248 data_ = NULL;
249 block_files_.CloseFiles();
250 rankings_.Reset();
251 init_ = false;
252 restarted_ = true;
253 }
254
~BackendImpl()255 BackendImpl::~BackendImpl() {
256 if (user_flags_ & kNoRandom) {
257 // This is a unit test, so we want to be strict about not leaking entries
258 // and completing all the work.
259 background_queue_.WaitForPendingIO();
260 } else {
261 // This is most likely not a test, so we want to do as little work as
262 // possible at this time, at the price of leaving dirty entries behind.
263 background_queue_.DropPendingIO();
264 }
265
266 if (background_queue_.BackgroundIsCurrentThread()) {
267 // Unit tests may use the same thread for everything.
268 CleanupCache();
269 } else {
270 background_queue_.background_thread()->PostTask(
271 FROM_HERE, base::Bind(&FinalCleanupCallback, base::Unretained(this)));
272 // http://crbug.com/74623
273 base::ThreadRestrictions::ScopedAllowWait allow_wait;
274 done_.Wait();
275 }
276 }
277
CleanupCache()278 void BackendImpl::CleanupCache() {
279 Trace("Backend Cleanup");
280 eviction_.Stop();
281 timer_.reset();
282
283 if (init_) {
284 StoreStats();
285 if (data_)
286 data_->header.crash = 0;
287
288 if (user_flags_ & kNoRandom) {
289 // This is a net_unittest, verify that we are not 'leaking' entries.
290 File::WaitForPendingIO(&num_pending_io_);
291 DCHECK(!num_refs_);
292 } else {
293 File::DropPendingIO();
294 }
295 }
296 block_files_.CloseFiles();
297 FlushIndex();
298 index_ = NULL;
299 ptr_factory_.InvalidateWeakPtrs();
300 done_.Signal();
301 }
302
GetFileName(Addr address) const303 base::FilePath BackendImpl::GetFileName(Addr address) const {
304 if (!address.is_separate_file() || !address.is_initialized()) {
305 NOTREACHED();
306 return base::FilePath();
307 }
308
309 std::string tmp = base::StringPrintf("f_%06x", address.FileNumber());
310 return path_.AppendASCII(tmp);
311 }
312
313 // We just created a new file so we're going to write the header and set the
314 // file length to include the hash table (zero filled).
CreateBackingStore(disk_cache::File * file)315 bool BackendImpl::CreateBackingStore(disk_cache::File* file) {
316 AdjustMaxCacheSize(0);
317
318 IndexHeader header;
319 header.table_len = DesiredIndexTableLen(max_size_);
320
321 // We need file version 2.1 for the new eviction algorithm.
322 if (new_eviction_)
323 header.version = 0x20001;
324
325 header.create_time = Time::Now().ToInternalValue();
326
327 if (!file->Write(&header, sizeof(header), 0))
328 return false;
329
330 return file->SetLength(GetIndexSize(header.table_len));
331 }
332
InitBackingStore(bool * file_created)333 bool BackendImpl::InitBackingStore(bool* file_created) {
334 if (!base::CreateDirectory(path_))
335 return false;
336
337 base::FilePath index_name = path_.AppendASCII(kIndexName);
338
339 int flags = base::PLATFORM_FILE_READ |
340 base::PLATFORM_FILE_WRITE |
341 base::PLATFORM_FILE_OPEN_ALWAYS |
342 base::PLATFORM_FILE_EXCLUSIVE_WRITE;
343 scoped_refptr<disk_cache::File> file(new disk_cache::File(
344 base::CreatePlatformFile(index_name, flags, file_created, NULL)));
345
346 if (!file->IsValid())
347 return false;
348
349 bool ret = true;
350 if (*file_created)
351 ret = CreateBackingStore(file.get());
352
353 file = NULL;
354 if (!ret)
355 return false;
356
357 index_ = new MappedFile();
358 data_ = reinterpret_cast<Index*>(index_->Init(index_name, 0));
359 if (!data_) {
360 LOG(ERROR) << "Unable to map Index file";
361 return false;
362 }
363
364 if (index_->GetLength() < sizeof(Index)) {
365 // We verify this again on CheckIndex() but it's easier to make sure now
366 // that the header is there.
367 LOG(ERROR) << "Corrupt Index file";
368 return false;
369 }
370
371 return true;
372 }
373
ReportError(int error)374 void BackendImpl::ReportError(int error) {
375 STRESS_DCHECK(!error || error == ERR_PREVIOUS_CRASH ||
376 error == ERR_CACHE_CREATED);
377
378 // We transmit positive numbers, instead of direct error codes.
379 DCHECK_LE(error, 0);
380 CACHE_UMA(CACHE_ERROR, "Error", 0, error * -1);
381 }
382
383
CheckIndex()384 bool BackendImpl::CheckIndex() {
385 DCHECK(data_);
386
387 size_t current_size = index_->GetLength();
388 if (current_size < sizeof(Index)) {
389 LOG(ERROR) << "Corrupt Index file";
390 return false;
391 }
392
393 if (new_eviction_) {
394 // We support versions 2.0 and 2.1, upgrading 2.0 to 2.1.
395 if (kIndexMagic != data_->header.magic ||
396 kCurrentVersion >> 16 != data_->header.version >> 16) {
397 LOG(ERROR) << "Invalid file version or magic";
398 return false;
399 }
400 if (kCurrentVersion == data_->header.version) {
401 // We need file version 2.1 for the new eviction algorithm.
402 UpgradeTo2_1();
403 }
404 } else {
405 if (kIndexMagic != data_->header.magic ||
406 kCurrentVersion != data_->header.version) {
407 LOG(ERROR) << "Invalid file version or magic";
408 return false;
409 }
410 }
411
412 if (!data_->header.table_len) {
413 LOG(ERROR) << "Invalid table size";
414 return false;
415 }
416
417 if (current_size < GetIndexSize(data_->header.table_len) ||
418 data_->header.table_len & (kBaseTableLen - 1)) {
419 LOG(ERROR) << "Corrupt Index file";
420 return false;
421 }
422
423 AdjustMaxCacheSize(data_->header.table_len);
424
425 #if !defined(NET_BUILD_STRESS_CACHE)
426 if (data_->header.num_bytes < 0 ||
427 (max_size_ < kint32max - kDefaultCacheSize &&
428 data_->header.num_bytes > max_size_ + kDefaultCacheSize)) {
429 LOG(ERROR) << "Invalid cache (current) size";
430 return false;
431 }
432 #endif
433
434 if (data_->header.num_entries < 0) {
435 LOG(ERROR) << "Invalid number of entries";
436 return false;
437 }
438
439 if (!mask_)
440 mask_ = data_->header.table_len - 1;
441
442 // Load the table into memory with a single read.
443 scoped_ptr<char[]> buf(new char[current_size]);
444 return index_->Read(buf.get(), current_size, 0);
445 }
446
InitStats()447 bool BackendImpl::InitStats() {
448 Addr address(data_->header.stats);
449 int size = stats_.StorageSize();
450
451 if (!address.is_initialized()) {
452 FileType file_type = Addr::RequiredFileType(size);
453 DCHECK_NE(file_type, EXTERNAL);
454 int num_blocks = Addr::RequiredBlocks(size, file_type);
455
456 if (!CreateBlock(file_type, num_blocks, &address))
457 return false;
458 return stats_.Init(NULL, 0, address);
459 }
460
461 if (!address.is_block_file()) {
462 NOTREACHED();
463 return false;
464 }
465
466 // Load the required data.
467 size = address.num_blocks() * address.BlockSize();
468 MappedFile* file = File(address);
469 if (!file)
470 return false;
471
472 scoped_ptr<char[]> data(new char[size]);
473 size_t offset = address.start_block() * address.BlockSize() +
474 kBlockHeaderSize;
475 if (!file->Read(data.get(), size, offset))
476 return false;
477
478 if (!stats_.Init(data.get(), size, address))
479 return false;
480 if (cache_type_ == net::DISK_CACHE && ShouldReportAgain())
481 stats_.InitSizeHistogram();
482 return true;
483 }
484
485 } // namespace disk_cache
486