1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "chrome/browser/chromeos/drive/resource_metadata_storage.h"
6
7 #include "base/bind.h"
8 #include "base/file_util.h"
9 #include "base/location.h"
10 #include "base/logging.h"
11 #include "base/metrics/histogram.h"
12 #include "base/metrics/sparse_histogram.h"
13 #include "base/sequenced_task_runner.h"
14 #include "base/threading/thread_restrictions.h"
15 #include "chrome/browser/chromeos/drive/drive.pb.h"
16 #include "third_party/leveldatabase/env_chromium.h"
17 #include "third_party/leveldatabase/src/include/leveldb/db.h"
18 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
19
20 namespace drive {
21 namespace internal {
22
23 namespace {
24
25 // Enum to describe DB initialization status.
26 enum DBInitStatus {
27 DB_INIT_SUCCESS,
28 DB_INIT_NOT_FOUND,
29 DB_INIT_CORRUPTION,
30 DB_INIT_IO_ERROR,
31 DB_INIT_FAILED,
32 DB_INIT_INCOMPATIBLE,
33 DB_INIT_BROKEN,
34 DB_INIT_OPENED_EXISTING_DB,
35 DB_INIT_CREATED_NEW_DB,
36 DB_INIT_REPLACED_EXISTING_DB_WITH_NEW_DB,
37 DB_INIT_MAX_VALUE,
38 };
39
40 // The name of the DB which stores the metadata.
41 const base::FilePath::CharType kResourceMapDBName[] =
42 FILE_PATH_LITERAL("resource_metadata_resource_map.db");
43
44 // The name of the DB which couldn't be opened, but is preserved just in case.
45 const base::FilePath::CharType kPreservedResourceMapDBName[] =
46 FILE_PATH_LITERAL("resource_metadata_preserved_resource_map.db");
47
48 // The name of the DB which couldn't be opened, and was replaced with a new one.
49 const base::FilePath::CharType kTrashedResourceMapDBName[] =
50 FILE_PATH_LITERAL("resource_metadata_trashed_resource_map.db");
51
52 // Meant to be a character which never happen to be in real IDs.
53 const char kDBKeyDelimeter = '\0';
54
55 // String used as a suffix of a key for a cache entry.
56 const char kCacheEntryKeySuffix[] = "CACHE";
57
58 // String used as a prefix of a key for a resource-ID-to-local-ID entry.
59 const char kIdEntryKeyPrefix[] = "ID";
60
61 // Returns a string to be used as the key for the header.
GetHeaderDBKey()62 std::string GetHeaderDBKey() {
63 std::string key;
64 key.push_back(kDBKeyDelimeter);
65 key.append("HEADER");
66 return key;
67 }
68
69 // Returns true if |key| is a key for a child entry.
IsChildEntryKey(const leveldb::Slice & key)70 bool IsChildEntryKey(const leveldb::Slice& key) {
71 return !key.empty() && key[key.size() - 1] == kDBKeyDelimeter;
72 }
73
74 // Returns true if |key| is a key for a cache entry.
IsCacheEntryKey(const leveldb::Slice & key)75 bool IsCacheEntryKey(const leveldb::Slice& key) {
76 // A cache entry key should end with |kDBKeyDelimeter + kCacheEntryKeySuffix|.
77 const leveldb::Slice expected_suffix(kCacheEntryKeySuffix,
78 arraysize(kCacheEntryKeySuffix) - 1);
79 if (key.size() < 1 + expected_suffix.size() ||
80 key[key.size() - expected_suffix.size() - 1] != kDBKeyDelimeter)
81 return false;
82
83 const leveldb::Slice key_substring(
84 key.data() + key.size() - expected_suffix.size(), expected_suffix.size());
85 return key_substring.compare(expected_suffix) == 0;
86 }
87
88 // Returns ID extracted from a cache entry key.
GetIdFromCacheEntryKey(const leveldb::Slice & key)89 std::string GetIdFromCacheEntryKey(const leveldb::Slice& key) {
90 DCHECK(IsCacheEntryKey(key));
91 // Drop the suffix |kDBKeyDelimeter + kCacheEntryKeySuffix| from the key.
92 const size_t kSuffixLength = arraysize(kCacheEntryKeySuffix) - 1;
93 const int id_length = key.size() - 1 - kSuffixLength;
94 return std::string(key.data(), id_length);
95 }
96
97 // Returns a string to be used as a key for a resource-ID-to-local-ID entry.
GetIdEntryKey(const std::string & resource_id)98 std::string GetIdEntryKey(const std::string& resource_id) {
99 std::string key;
100 key.push_back(kDBKeyDelimeter);
101 key.append(kIdEntryKeyPrefix);
102 key.push_back(kDBKeyDelimeter);
103 key.append(resource_id);
104 return key;
105 }
106
107 // Returns true if |key| is a key for a resource-ID-to-local-ID entry.
IsIdEntryKey(const leveldb::Slice & key)108 bool IsIdEntryKey(const leveldb::Slice& key) {
109 // A resource-ID-to-local-ID entry key should start with
110 // |kDBKeyDelimeter + kIdEntryKeyPrefix + kDBKeyDelimeter|.
111 const leveldb::Slice expected_prefix(kIdEntryKeyPrefix,
112 arraysize(kIdEntryKeyPrefix) - 1);
113 if (key.size() < 2 + expected_prefix.size())
114 return false;
115 const leveldb::Slice key_substring(key.data() + 1, expected_prefix.size());
116 return key[0] == kDBKeyDelimeter &&
117 key_substring.compare(expected_prefix) == 0 &&
118 key[expected_prefix.size() + 1] == kDBKeyDelimeter;
119 }
120
121 // Returns the resource ID extracted from a resource-ID-to-local-ID entry key.
GetResourceIdFromIdEntryKey(const leveldb::Slice & key)122 std::string GetResourceIdFromIdEntryKey(const leveldb::Slice& key) {
123 DCHECK(IsIdEntryKey(key));
124 // Drop the prefix |kDBKeyDelimeter + kIdEntryKeyPrefix + kDBKeyDelimeter|
125 // from the key.
126 const size_t kPrefixLength = arraysize(kIdEntryKeyPrefix) - 1;
127 const int offset = kPrefixLength + 2;
128 return std::string(key.data() + offset, key.size() - offset);
129 }
130
131 // Converts leveldb::Status to DBInitStatus.
LevelDBStatusToDBInitStatus(const leveldb::Status & status)132 DBInitStatus LevelDBStatusToDBInitStatus(const leveldb::Status& status) {
133 if (status.ok())
134 return DB_INIT_SUCCESS;
135 if (status.IsNotFound())
136 return DB_INIT_NOT_FOUND;
137 if (status.IsCorruption())
138 return DB_INIT_CORRUPTION;
139 if (status.IsIOError())
140 return DB_INIT_IO_ERROR;
141 return DB_INIT_FAILED;
142 }
143
144 // Converts leveldb::Status to FileError.
LevelDBStatusToFileError(const leveldb::Status & status)145 FileError LevelDBStatusToFileError(const leveldb::Status& status) {
146 if (status.ok())
147 return FILE_ERROR_OK;
148 if (status.IsNotFound())
149 return FILE_ERROR_NOT_FOUND;
150 if (leveldb_env::IndicatesDiskFull(status))
151 return FILE_ERROR_NO_LOCAL_SPACE;
152 return FILE_ERROR_FAILED;
153 }
154
GetDefaultHeaderEntry()155 ResourceMetadataHeader GetDefaultHeaderEntry() {
156 ResourceMetadataHeader header;
157 header.set_version(ResourceMetadataStorage::kDBVersion);
158 return header;
159 }
160
MoveIfPossible(const base::FilePath & from,const base::FilePath & to)161 bool MoveIfPossible(const base::FilePath& from, const base::FilePath& to) {
162 return !base::PathExists(from) || base::Move(from, to);
163 }
164
165 } // namespace
166
Iterator(scoped_ptr<leveldb::Iterator> it)167 ResourceMetadataStorage::Iterator::Iterator(scoped_ptr<leveldb::Iterator> it)
168 : it_(it.Pass()) {
169 base::ThreadRestrictions::AssertIOAllowed();
170 DCHECK(it_);
171
172 // Skip the header entry.
173 // Note: The header entry comes before all other entries because its key
174 // starts with kDBKeyDelimeter. (i.e. '\0')
175 it_->Seek(leveldb::Slice(GetHeaderDBKey()));
176
177 Advance();
178 }
179
~Iterator()180 ResourceMetadataStorage::Iterator::~Iterator() {
181 base::ThreadRestrictions::AssertIOAllowed();
182 }
183
IsAtEnd() const184 bool ResourceMetadataStorage::Iterator::IsAtEnd() const {
185 base::ThreadRestrictions::AssertIOAllowed();
186 return !it_->Valid();
187 }
188
GetID() const189 std::string ResourceMetadataStorage::Iterator::GetID() const {
190 return it_->key().ToString();
191 }
192
GetValue() const193 const ResourceEntry& ResourceMetadataStorage::Iterator::GetValue() const {
194 base::ThreadRestrictions::AssertIOAllowed();
195 DCHECK(!IsAtEnd());
196 return entry_;
197 }
198
Advance()199 void ResourceMetadataStorage::Iterator::Advance() {
200 base::ThreadRestrictions::AssertIOAllowed();
201 DCHECK(!IsAtEnd());
202
203 for (it_->Next() ; it_->Valid(); it_->Next()) {
204 if (!IsChildEntryKey(it_->key()) &&
205 !IsIdEntryKey(it_->key()) &&
206 entry_.ParseFromArray(it_->value().data(), it_->value().size())) {
207 break;
208 }
209 }
210 }
211
HasError() const212 bool ResourceMetadataStorage::Iterator::HasError() const {
213 base::ThreadRestrictions::AssertIOAllowed();
214 return !it_->status().ok();
215 }
216
217 // static
UpgradeOldDB(const base::FilePath & directory_path,const ResourceIdCanonicalizer & id_canonicalizer)218 bool ResourceMetadataStorage::UpgradeOldDB(
219 const base::FilePath& directory_path,
220 const ResourceIdCanonicalizer& id_canonicalizer) {
221 base::ThreadRestrictions::AssertIOAllowed();
222 COMPILE_ASSERT(
223 kDBVersion == 13,
224 db_version_and_this_function_should_be_updated_at_the_same_time);
225
226 const base::FilePath resource_map_path =
227 directory_path.Append(kResourceMapDBName);
228 const base::FilePath preserved_resource_map_path =
229 directory_path.Append(kPreservedResourceMapDBName);
230
231 if (base::PathExists(preserved_resource_map_path)) {
232 // Preserved DB is found. The previous attempt to create a new DB should not
233 // be successful. Discard the imperfect new DB and restore the old DB.
234 if (!base::DeleteFile(resource_map_path, false /* recursive */) ||
235 !base::Move(preserved_resource_map_path, resource_map_path))
236 return false;
237 }
238
239 if (!base::PathExists(resource_map_path))
240 return false;
241
242 // Open DB.
243 leveldb::DB* db = NULL;
244 leveldb::Options options;
245 options.max_open_files = 0; // Use minimum.
246 options.create_if_missing = false;
247 if (!leveldb::DB::Open(options, resource_map_path.AsUTF8Unsafe(), &db).ok())
248 return false;
249 scoped_ptr<leveldb::DB> resource_map(db);
250
251 // Check DB version.
252 std::string serialized_header;
253 ResourceMetadataHeader header;
254 if (!resource_map->Get(leveldb::ReadOptions(),
255 leveldb::Slice(GetHeaderDBKey()),
256 &serialized_header).ok() ||
257 !header.ParseFromString(serialized_header))
258 return false;
259 UMA_HISTOGRAM_SPARSE_SLOWLY("Drive.MetadataDBVersionBeforeUpgradeCheck",
260 header.version());
261
262 if (header.version() == kDBVersion) {
263 // Before r272134, UpgradeOldDB() was not deleting unused ID entries.
264 // Delete unused ID entries to fix crbug.com/374648.
265 std::set<std::string> used_ids;
266
267 scoped_ptr<leveldb::Iterator> it(
268 resource_map->NewIterator(leveldb::ReadOptions()));
269 it->Seek(leveldb::Slice(GetHeaderDBKey()));
270 it->Next();
271 for (; it->Valid(); it->Next()) {
272 if (IsCacheEntryKey(it->key())) {
273 used_ids.insert(GetIdFromCacheEntryKey(it->key()));
274 } else if (!IsChildEntryKey(it->key()) && !IsIdEntryKey(it->key())) {
275 used_ids.insert(it->key().ToString());
276 }
277 }
278 if (!it->status().ok())
279 return false;
280
281 leveldb::WriteBatch batch;
282 for (it->SeekToFirst(); it->Valid(); it->Next()) {
283 if (IsIdEntryKey(it->key()) && !used_ids.count(it->value().ToString()))
284 batch.Delete(it->key());
285 }
286 if (!it->status().ok())
287 return false;
288
289 return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
290 } else if (header.version() < 6) { // Too old, nothing can be done.
291 return false;
292 } else if (header.version() < 11) { // Cache entries can be reused.
293 leveldb::ReadOptions options;
294 options.verify_checksums = true;
295 scoped_ptr<leveldb::Iterator> it(resource_map->NewIterator(options));
296
297 leveldb::WriteBatch batch;
298 // First, remove all entries.
299 for (it->SeekToFirst(); it->Valid(); it->Next())
300 batch.Delete(it->key());
301
302 // Put ID entries and cache entries.
303 for (it->SeekToFirst(); it->Valid(); it->Next()) {
304 if (IsCacheEntryKey(it->key())) {
305 FileCacheEntry cache_entry;
306 if (!cache_entry.ParseFromArray(it->value().data(), it->value().size()))
307 return false;
308
309 // The resource ID might be in old WAPI format. We need to canonicalize
310 // to the format of API service currently in use.
311 const std::string& id = GetIdFromCacheEntryKey(it->key());
312 const std::string& id_new = id_canonicalizer.Run(id);
313
314 // Before v11, resource ID was directly used as local ID. Such entries
315 // can be migrated by adding an identity ID mapping.
316 batch.Put(GetIdEntryKey(id_new), id_new);
317
318 // Put cache state into a ResourceEntry.
319 ResourceEntry entry;
320 entry.set_local_id(id_new);
321 entry.set_resource_id(id_new);
322 *entry.mutable_file_specific_info()->mutable_cache_state() =
323 cache_entry;
324
325 std::string serialized_entry;
326 if (!entry.SerializeToString(&serialized_entry)) {
327 DLOG(ERROR) << "Failed to serialize the entry: " << id;
328 return false;
329 }
330 batch.Put(id_new, serialized_entry);
331 }
332 }
333 if (!it->status().ok())
334 return false;
335
336 // Put header with the latest version number.
337 std::string serialized_header;
338 if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
339 return false;
340 batch.Put(GetHeaderDBKey(), serialized_header);
341
342 return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
343 } else if (header.version() < 12) { // Cache and ID map entries are reusable.
344 leveldb::ReadOptions options;
345 options.verify_checksums = true;
346 scoped_ptr<leveldb::Iterator> it(resource_map->NewIterator(options));
347
348 // First, get the set of local IDs associated with cache entries.
349 std::set<std::string> cached_entry_ids;
350 for (it->SeekToFirst(); it->Valid(); it->Next()) {
351 if (IsCacheEntryKey(it->key()))
352 cached_entry_ids.insert(GetIdFromCacheEntryKey(it->key()));
353 }
354 if (!it->status().ok())
355 return false;
356
357 // Remove all entries except used ID entries.
358 leveldb::WriteBatch batch;
359 std::map<std::string, std::string> local_id_to_resource_id;
360 for (it->SeekToFirst(); it->Valid(); it->Next()) {
361 const bool is_used_id = IsIdEntryKey(it->key()) &&
362 cached_entry_ids.count(it->value().ToString());
363 if (is_used_id) {
364 local_id_to_resource_id[it->value().ToString()] =
365 GetResourceIdFromIdEntryKey(it->key());
366 } else {
367 batch.Delete(it->key());
368 }
369 }
370 if (!it->status().ok())
371 return false;
372
373 // Put cache entries.
374 for (it->SeekToFirst(); it->Valid(); it->Next()) {
375 if (IsCacheEntryKey(it->key())) {
376 const std::string& id = GetIdFromCacheEntryKey(it->key());
377
378 std::map<std::string, std::string>::const_iterator iter_resource_id =
379 local_id_to_resource_id.find(id);
380 if (iter_resource_id == local_id_to_resource_id.end())
381 continue;
382
383 FileCacheEntry cache_entry;
384 if (!cache_entry.ParseFromArray(it->value().data(), it->value().size()))
385 return false;
386
387 // Put cache state into a ResourceEntry.
388 ResourceEntry entry;
389 entry.set_local_id(id);
390 entry.set_resource_id(iter_resource_id->second);
391 *entry.mutable_file_specific_info()->mutable_cache_state() =
392 cache_entry;
393
394 std::string serialized_entry;
395 if (!entry.SerializeToString(&serialized_entry)) {
396 DLOG(ERROR) << "Failed to serialize the entry: " << id;
397 return false;
398 }
399 batch.Put(id, serialized_entry);
400 }
401 }
402 if (!it->status().ok())
403 return false;
404
405 // Put header with the latest version number.
406 std::string serialized_header;
407 if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
408 return false;
409 batch.Put(GetHeaderDBKey(), serialized_header);
410
411 return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
412 } else if (header.version() < 13) { // Reuse all entries.
413 leveldb::ReadOptions options;
414 options.verify_checksums = true;
415 scoped_ptr<leveldb::Iterator> it(resource_map->NewIterator(options));
416
417 // First, get local ID to resource ID map.
418 std::map<std::string, std::string> local_id_to_resource_id;
419 for (it->SeekToFirst(); it->Valid(); it->Next()) {
420 if (IsIdEntryKey(it->key())) {
421 local_id_to_resource_id[it->value().ToString()] =
422 GetResourceIdFromIdEntryKey(it->key());
423 }
424 }
425 if (!it->status().ok())
426 return false;
427
428 leveldb::WriteBatch batch;
429 // Merge cache entries to ResourceEntry.
430 for (it->SeekToFirst(); it->Valid(); it->Next()) {
431 if (IsCacheEntryKey(it->key())) {
432 const std::string& id = GetIdFromCacheEntryKey(it->key());
433
434 FileCacheEntry cache_entry;
435 if (!cache_entry.ParseFromArray(it->value().data(), it->value().size()))
436 return false;
437
438 std::string serialized_entry;
439 leveldb::Status status = resource_map->Get(options,
440 leveldb::Slice(id),
441 &serialized_entry);
442
443 std::map<std::string, std::string>::const_iterator iter_resource_id =
444 local_id_to_resource_id.find(id);
445
446 // No need to keep cache-only entries without resource ID.
447 if (status.IsNotFound() &&
448 iter_resource_id == local_id_to_resource_id.end())
449 continue;
450
451 ResourceEntry entry;
452 if (status.ok()) {
453 if (!entry.ParseFromString(serialized_entry))
454 return false;
455 } else if (status.IsNotFound()) {
456 entry.set_local_id(id);
457 entry.set_resource_id(iter_resource_id->second);
458 } else {
459 DLOG(ERROR) << "Failed to get the entry: " << id;
460 return false;
461 }
462 *entry.mutable_file_specific_info()->mutable_cache_state() =
463 cache_entry;
464
465 if (!entry.SerializeToString(&serialized_entry)) {
466 DLOG(ERROR) << "Failed to serialize the entry: " << id;
467 return false;
468 }
469 batch.Delete(it->key());
470 batch.Put(id, serialized_entry);
471 }
472 }
473 if (!it->status().ok())
474 return false;
475
476 // Put header with the latest version number.
477 header.set_version(ResourceMetadataStorage::kDBVersion);
478 std::string serialized_header;
479 if (!header.SerializeToString(&serialized_header))
480 return false;
481 batch.Put(GetHeaderDBKey(), serialized_header);
482
483 return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
484 }
485
486 LOG(WARNING) << "Unexpected DB version: " << header.version();
487 return false;
488 }
489
ResourceMetadataStorage(const base::FilePath & directory_path,base::SequencedTaskRunner * blocking_task_runner)490 ResourceMetadataStorage::ResourceMetadataStorage(
491 const base::FilePath& directory_path,
492 base::SequencedTaskRunner* blocking_task_runner)
493 : directory_path_(directory_path),
494 cache_file_scan_is_needed_(true),
495 blocking_task_runner_(blocking_task_runner) {
496 }
497
Destroy()498 void ResourceMetadataStorage::Destroy() {
499 blocking_task_runner_->PostTask(
500 FROM_HERE,
501 base::Bind(&ResourceMetadataStorage::DestroyOnBlockingPool,
502 base::Unretained(this)));
503 }
504
Initialize()505 bool ResourceMetadataStorage::Initialize() {
506 base::ThreadRestrictions::AssertIOAllowed();
507
508 resource_map_.reset();
509
510 const base::FilePath resource_map_path =
511 directory_path_.Append(kResourceMapDBName);
512 const base::FilePath preserved_resource_map_path =
513 directory_path_.Append(kPreservedResourceMapDBName);
514 const base::FilePath trashed_resource_map_path =
515 directory_path_.Append(kTrashedResourceMapDBName);
516
517 // Discard unneeded DBs.
518 if (!base::DeleteFile(preserved_resource_map_path, true /* recursive */) ||
519 !base::DeleteFile(trashed_resource_map_path, true /* recursive */)) {
520 LOG(ERROR) << "Failed to remove unneeded DBs.";
521 return false;
522 }
523
524 // Try to open the existing DB.
525 leveldb::DB* db = NULL;
526 leveldb::Options options;
527 options.max_open_files = 0; // Use minimum.
528 options.create_if_missing = false;
529
530 DBInitStatus open_existing_result = DB_INIT_NOT_FOUND;
531 leveldb::Status status;
532 if (base::PathExists(resource_map_path)) {
533 status = leveldb::DB::Open(options, resource_map_path.AsUTF8Unsafe(), &db);
534 open_existing_result = LevelDBStatusToDBInitStatus(status);
535 }
536
537 if (open_existing_result == DB_INIT_SUCCESS) {
538 resource_map_.reset(db);
539
540 // Check the validity of existing DB.
541 int db_version = -1;
542 ResourceMetadataHeader header;
543 if (GetHeader(&header) == FILE_ERROR_OK)
544 db_version = header.version();
545
546 bool should_discard_db = true;
547 if (db_version != kDBVersion) {
548 open_existing_result = DB_INIT_INCOMPATIBLE;
549 DVLOG(1) << "Reject incompatible DB.";
550 } else if (!CheckValidity()) {
551 open_existing_result = DB_INIT_BROKEN;
552 LOG(ERROR) << "Reject invalid DB.";
553 } else {
554 should_discard_db = false;
555 }
556
557 if (should_discard_db)
558 resource_map_.reset();
559 else
560 cache_file_scan_is_needed_ = false;
561 }
562
563 UMA_HISTOGRAM_ENUMERATION("Drive.MetadataDBOpenExistingResult",
564 open_existing_result,
565 DB_INIT_MAX_VALUE);
566
567 DBInitStatus init_result = DB_INIT_OPENED_EXISTING_DB;
568
569 // Failed to open the existing DB, create new DB.
570 if (!resource_map_) {
571 // Move the existing DB to the preservation path. The moved old DB is
572 // deleted once the new DB creation succeeds, or is restored later in
573 // UpgradeOldDB() when the creation fails.
574 MoveIfPossible(resource_map_path, preserved_resource_map_path);
575
576 // Create DB.
577 options.max_open_files = 0; // Use minimum.
578 options.create_if_missing = true;
579 options.error_if_exists = true;
580
581 status = leveldb::DB::Open(options, resource_map_path.AsUTF8Unsafe(), &db);
582 if (status.ok()) {
583 resource_map_.reset(db);
584
585 // Set up header and trash the old DB.
586 if (PutHeader(GetDefaultHeaderEntry()) == FILE_ERROR_OK &&
587 MoveIfPossible(preserved_resource_map_path,
588 trashed_resource_map_path)) {
589 init_result = open_existing_result == DB_INIT_NOT_FOUND ?
590 DB_INIT_CREATED_NEW_DB : DB_INIT_REPLACED_EXISTING_DB_WITH_NEW_DB;
591 } else {
592 init_result = DB_INIT_FAILED;
593 resource_map_.reset();
594 }
595 } else {
596 LOG(ERROR) << "Failed to create resource map DB: " << status.ToString();
597 init_result = LevelDBStatusToDBInitStatus(status);
598 }
599 }
600
601 UMA_HISTOGRAM_ENUMERATION("Drive.MetadataDBInitResult",
602 init_result,
603 DB_INIT_MAX_VALUE);
604 return resource_map_;
605 }
606
RecoverCacheInfoFromTrashedResourceMap(RecoveredCacheInfoMap * out_info)607 void ResourceMetadataStorage::RecoverCacheInfoFromTrashedResourceMap(
608 RecoveredCacheInfoMap* out_info) {
609 const base::FilePath trashed_resource_map_path =
610 directory_path_.Append(kTrashedResourceMapDBName);
611
612 if (!base::PathExists(trashed_resource_map_path))
613 return;
614
615 leveldb::Options options;
616 options.max_open_files = 0; // Use minimum.
617 options.create_if_missing = false;
618
619 // Trashed DB may be broken, repair it first.
620 leveldb::Status status;
621 status = leveldb::RepairDB(trashed_resource_map_path.AsUTF8Unsafe(), options);
622 if (!status.ok()) {
623 LOG(ERROR) << "Failed to repair trashed DB: " << status.ToString();
624 return;
625 }
626
627 // Open it.
628 leveldb::DB* db = NULL;
629 status = leveldb::DB::Open(options, trashed_resource_map_path.AsUTF8Unsafe(),
630 &db);
631 if (!status.ok()) {
632 LOG(ERROR) << "Failed to open trashed DB: " << status.ToString();
633 return;
634 }
635 scoped_ptr<leveldb::DB> resource_map(db);
636
637 // Check DB version.
638 std::string serialized_header;
639 ResourceMetadataHeader header;
640 if (!resource_map->Get(leveldb::ReadOptions(),
641 leveldb::Slice(GetHeaderDBKey()),
642 &serialized_header).ok() ||
643 !header.ParseFromString(serialized_header) ||
644 header.version() != kDBVersion) {
645 LOG(ERROR) << "Incompatible DB version: " << header.version();
646 return;
647 }
648
649 // Collect cache entries.
650 scoped_ptr<leveldb::Iterator> it(
651 resource_map->NewIterator(leveldb::ReadOptions()));
652 for (it->SeekToFirst(); it->Valid(); it->Next()) {
653 if (!IsChildEntryKey(it->key()) &&
654 !IsIdEntryKey(it->key())) {
655 const std::string id = it->key().ToString();
656 ResourceEntry entry;
657 if (entry.ParseFromArray(it->value().data(), it->value().size()) &&
658 entry.file_specific_info().has_cache_state()) {
659 RecoveredCacheInfo* info = &(*out_info)[id];
660 info->is_dirty = entry.file_specific_info().cache_state().is_dirty();
661 info->md5 = entry.file_specific_info().cache_state().md5();
662 info->title = entry.title();
663 }
664 }
665 }
666 }
667
SetLargestChangestamp(int64 largest_changestamp)668 FileError ResourceMetadataStorage::SetLargestChangestamp(
669 int64 largest_changestamp) {
670 base::ThreadRestrictions::AssertIOAllowed();
671
672 ResourceMetadataHeader header;
673 FileError error = GetHeader(&header);
674 if (error != FILE_ERROR_OK) {
675 DLOG(ERROR) << "Failed to get the header.";
676 return error;
677 }
678 header.set_largest_changestamp(largest_changestamp);
679 return PutHeader(header);
680 }
681
GetLargestChangestamp(int64 * largest_changestamp)682 FileError ResourceMetadataStorage::GetLargestChangestamp(
683 int64* largest_changestamp) {
684 base::ThreadRestrictions::AssertIOAllowed();
685 ResourceMetadataHeader header;
686 FileError error = GetHeader(&header);
687 if (error != FILE_ERROR_OK) {
688 DLOG(ERROR) << "Failed to get the header.";
689 return error;
690 }
691 *largest_changestamp = header.largest_changestamp();
692 return FILE_ERROR_OK;
693 }
694
PutEntry(const ResourceEntry & entry)695 FileError ResourceMetadataStorage::PutEntry(const ResourceEntry& entry) {
696 base::ThreadRestrictions::AssertIOAllowed();
697
698 const std::string& id = entry.local_id();
699 DCHECK(!id.empty());
700
701 // Try to get existing entry.
702 std::string serialized_entry;
703 leveldb::Status status = resource_map_->Get(leveldb::ReadOptions(),
704 leveldb::Slice(id),
705 &serialized_entry);
706 if (!status.ok() && !status.IsNotFound()) // Unexpected errors.
707 return LevelDBStatusToFileError(status);
708
709 ResourceEntry old_entry;
710 if (status.ok() && !old_entry.ParseFromString(serialized_entry))
711 return FILE_ERROR_FAILED;
712
713 // Construct write batch.
714 leveldb::WriteBatch batch;
715
716 // Remove from the old parent.
717 if (!old_entry.parent_local_id().empty()) {
718 batch.Delete(GetChildEntryKey(old_entry.parent_local_id(),
719 old_entry.base_name()));
720 }
721 // Add to the new parent.
722 if (!entry.parent_local_id().empty())
723 batch.Put(GetChildEntryKey(entry.parent_local_id(), entry.base_name()), id);
724
725 // Refresh resource-ID-to-local-ID mapping entry.
726 if (old_entry.resource_id() != entry.resource_id()) {
727 // Resource ID should not change.
728 DCHECK(old_entry.resource_id().empty() || entry.resource_id().empty());
729
730 if (!old_entry.resource_id().empty())
731 batch.Delete(GetIdEntryKey(old_entry.resource_id()));
732 if (!entry.resource_id().empty())
733 batch.Put(GetIdEntryKey(entry.resource_id()), id);
734 }
735
736 // Put the entry itself.
737 if (!entry.SerializeToString(&serialized_entry)) {
738 DLOG(ERROR) << "Failed to serialize the entry: " << id;
739 return FILE_ERROR_FAILED;
740 }
741 batch.Put(id, serialized_entry);
742
743 status = resource_map_->Write(leveldb::WriteOptions(), &batch);
744 return LevelDBStatusToFileError(status);
745 }
746
GetEntry(const std::string & id,ResourceEntry * out_entry)747 FileError ResourceMetadataStorage::GetEntry(const std::string& id,
748 ResourceEntry* out_entry) {
749 base::ThreadRestrictions::AssertIOAllowed();
750 DCHECK(!id.empty());
751
752 std::string serialized_entry;
753 const leveldb::Status status = resource_map_->Get(leveldb::ReadOptions(),
754 leveldb::Slice(id),
755 &serialized_entry);
756 if (!status.ok())
757 return LevelDBStatusToFileError(status);
758 if (!out_entry->ParseFromString(serialized_entry))
759 return FILE_ERROR_FAILED;
760 return FILE_ERROR_OK;
761 }
762
RemoveEntry(const std::string & id)763 FileError ResourceMetadataStorage::RemoveEntry(const std::string& id) {
764 base::ThreadRestrictions::AssertIOAllowed();
765 DCHECK(!id.empty());
766
767 ResourceEntry entry;
768 FileError error = GetEntry(id, &entry);
769 if (error != FILE_ERROR_OK)
770 return error;
771
772 leveldb::WriteBatch batch;
773
774 // Remove from the parent.
775 if (!entry.parent_local_id().empty())
776 batch.Delete(GetChildEntryKey(entry.parent_local_id(), entry.base_name()));
777
778 // Remove resource ID-local ID mapping entry.
779 if (!entry.resource_id().empty())
780 batch.Delete(GetIdEntryKey(entry.resource_id()));
781
782 // Remove the entry itself.
783 batch.Delete(id);
784
785 const leveldb::Status status = resource_map_->Write(leveldb::WriteOptions(),
786 &batch);
787 return LevelDBStatusToFileError(status);
788 }
789
790 scoped_ptr<ResourceMetadataStorage::Iterator>
GetIterator()791 ResourceMetadataStorage::GetIterator() {
792 base::ThreadRestrictions::AssertIOAllowed();
793
794 scoped_ptr<leveldb::Iterator> it(
795 resource_map_->NewIterator(leveldb::ReadOptions()));
796 return make_scoped_ptr(new Iterator(it.Pass()));
797 }
798
GetChild(const std::string & parent_id,const std::string & child_name,std::string * child_id)799 FileError ResourceMetadataStorage::GetChild(const std::string& parent_id,
800 const std::string& child_name,
801 std::string* child_id) {
802 base::ThreadRestrictions::AssertIOAllowed();
803 DCHECK(!parent_id.empty());
804 DCHECK(!child_name.empty());
805
806 const leveldb::Status status =
807 resource_map_->Get(
808 leveldb::ReadOptions(),
809 leveldb::Slice(GetChildEntryKey(parent_id, child_name)),
810 child_id);
811 return LevelDBStatusToFileError(status);
812 }
813
GetChildren(const std::string & parent_id,std::vector<std::string> * children)814 FileError ResourceMetadataStorage::GetChildren(
815 const std::string& parent_id,
816 std::vector<std::string>* children) {
817 base::ThreadRestrictions::AssertIOAllowed();
818 DCHECK(!parent_id.empty());
819
820 // Iterate over all entries with keys starting with |parent_id|.
821 scoped_ptr<leveldb::Iterator> it(
822 resource_map_->NewIterator(leveldb::ReadOptions()));
823 for (it->Seek(parent_id);
824 it->Valid() && it->key().starts_with(leveldb::Slice(parent_id));
825 it->Next()) {
826 if (IsChildEntryKey(it->key()))
827 children->push_back(it->value().ToString());
828 }
829 return LevelDBStatusToFileError(it->status());
830 }
831
RecoveredCacheInfo()832 ResourceMetadataStorage::RecoveredCacheInfo::RecoveredCacheInfo()
833 : is_dirty(false) {}
834
~RecoveredCacheInfo()835 ResourceMetadataStorage::RecoveredCacheInfo::~RecoveredCacheInfo() {}
836
GetIdByResourceId(const std::string & resource_id,std::string * out_id)837 FileError ResourceMetadataStorage::GetIdByResourceId(
838 const std::string& resource_id,
839 std::string* out_id) {
840 base::ThreadRestrictions::AssertIOAllowed();
841 DCHECK(!resource_id.empty());
842
843 const leveldb::Status status = resource_map_->Get(
844 leveldb::ReadOptions(),
845 leveldb::Slice(GetIdEntryKey(resource_id)),
846 out_id);
847 return LevelDBStatusToFileError(status);
848 }
849
~ResourceMetadataStorage()850 ResourceMetadataStorage::~ResourceMetadataStorage() {
851 base::ThreadRestrictions::AssertIOAllowed();
852 }
853
DestroyOnBlockingPool()854 void ResourceMetadataStorage::DestroyOnBlockingPool() {
855 delete this;
856 }
857
858 // static
GetChildEntryKey(const std::string & parent_id,const std::string & child_name)859 std::string ResourceMetadataStorage::GetChildEntryKey(
860 const std::string& parent_id,
861 const std::string& child_name) {
862 DCHECK(!parent_id.empty());
863 DCHECK(!child_name.empty());
864
865 std::string key = parent_id;
866 key.push_back(kDBKeyDelimeter);
867 key.append(child_name);
868 key.push_back(kDBKeyDelimeter);
869 return key;
870 }
871
PutHeader(const ResourceMetadataHeader & header)872 FileError ResourceMetadataStorage::PutHeader(
873 const ResourceMetadataHeader& header) {
874 base::ThreadRestrictions::AssertIOAllowed();
875
876 std::string serialized_header;
877 if (!header.SerializeToString(&serialized_header)) {
878 DLOG(ERROR) << "Failed to serialize the header";
879 return FILE_ERROR_FAILED;
880 }
881
882 const leveldb::Status status = resource_map_->Put(
883 leveldb::WriteOptions(),
884 leveldb::Slice(GetHeaderDBKey()),
885 leveldb::Slice(serialized_header));
886 return LevelDBStatusToFileError(status);
887 }
888
GetHeader(ResourceMetadataHeader * header)889 FileError ResourceMetadataStorage::GetHeader(ResourceMetadataHeader* header) {
890 base::ThreadRestrictions::AssertIOAllowed();
891
892 std::string serialized_header;
893 const leveldb::Status status = resource_map_->Get(
894 leveldb::ReadOptions(),
895 leveldb::Slice(GetHeaderDBKey()),
896 &serialized_header);
897 if (!status.ok())
898 return LevelDBStatusToFileError(status);
899 return header->ParseFromString(serialized_header) ?
900 FILE_ERROR_OK : FILE_ERROR_FAILED;
901 }
902
CheckValidity()903 bool ResourceMetadataStorage::CheckValidity() {
904 base::ThreadRestrictions::AssertIOAllowed();
905
906 // Perform read with checksums verification enalbed.
907 leveldb::ReadOptions options;
908 options.verify_checksums = true;
909
910 scoped_ptr<leveldb::Iterator> it(resource_map_->NewIterator(options));
911 it->SeekToFirst();
912
913 // DB is organized like this:
914 //
915 // <key> : <value>
916 // "\0HEADER" : ResourceMetadataHeader
917 // "\0ID\0|resource ID 1|" : Local ID associated to resource ID 1.
918 // "\0ID\0|resource ID 2|" : Local ID associated to resource ID 2.
919 // ...
920 // "|ID of A|" : ResourceEntry for entry A.
921 // "|ID of A|\0|child name 1|\0" : ID of the 1st child entry of entry A.
922 // "|ID of A|\0|child name 2|\0" : ID of the 2nd child entry of entry A.
923 // ...
924 // "|ID of A|\0|child name n|\0" : ID of the nth child entry of entry A.
925 // "|ID of B|" : ResourceEntry for entry B.
926 // ...
927
928 // Check the header.
929 ResourceMetadataHeader header;
930 if (!it->Valid() ||
931 it->key() != GetHeaderDBKey() || // Header entry must come first.
932 !header.ParseFromArray(it->value().data(), it->value().size()) ||
933 header.version() != kDBVersion) {
934 DLOG(ERROR) << "Invalid header detected. version = " << header.version();
935 return false;
936 }
937
938 // Check all entries.
939 size_t num_entries_with_parent = 0;
940 size_t num_child_entries = 0;
941 ResourceEntry entry;
942 std::string serialized_entry;
943 std::string child_id;
944 for (it->Next(); it->Valid(); it->Next()) {
945 // Count child entries.
946 if (IsChildEntryKey(it->key())) {
947 ++num_child_entries;
948 continue;
949 }
950
951 // Check if resource-ID-to-local-ID mapping is stored correctly.
952 if (IsIdEntryKey(it->key())) {
953 leveldb::Status status = resource_map_->Get(
954 options, it->value(), &serialized_entry);
955 // Resource-ID-to-local-ID mapping without entry for the local ID is ok.
956 if (status.IsNotFound())
957 continue;
958 // When the entry exists, its resource ID must be consistent.
959 const bool ok = status.ok() &&
960 entry.ParseFromString(serialized_entry) &&
961 !entry.resource_id().empty() &&
962 leveldb::Slice(GetIdEntryKey(entry.resource_id())) == it->key();
963 if (!ok) {
964 DLOG(ERROR) << "Broken ID entry. status = " << status.ToString();
965 return false;
966 }
967 continue;
968 }
969
970 // Check if stored data is broken.
971 if (!entry.ParseFromArray(it->value().data(), it->value().size())) {
972 DLOG(ERROR) << "Broken entry detected";
973 return false;
974 }
975
976 if (leveldb::Slice(entry.local_id()) != it->key()) {
977 DLOG(ERROR) << "Wrong local ID.";
978 return false;
979 }
980
981 if (!entry.parent_local_id().empty()) {
982 // Check if the parent entry is stored.
983 leveldb::Status status = resource_map_->Get(
984 options,
985 leveldb::Slice(entry.parent_local_id()),
986 &serialized_entry);
987 if (!status.ok()) {
988 DLOG(ERROR) << "Can't get parent entry. status = " << status.ToString();
989 return false;
990 }
991
992 // Check if parent-child relationship is stored correctly.
993 status = resource_map_->Get(
994 options,
995 leveldb::Slice(GetChildEntryKey(entry.parent_local_id(),
996 entry.base_name())),
997 &child_id);
998 if (!status.ok() || leveldb::Slice(child_id) != it->key()) {
999 DLOG(ERROR) << "Child map is broken. status = " << status.ToString();
1000 return false;
1001 }
1002 ++num_entries_with_parent;
1003 }
1004 }
1005 if (!it->status().ok() || num_child_entries != num_entries_with_parent) {
1006 DLOG(ERROR) << "Error during checking resource map. status = "
1007 << it->status().ToString();
1008 return false;
1009 }
1010 return true;
1011 }
1012
1013 } // namespace internal
1014 } // namespace drive
1015