1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "chrome/browser/chromeos/drive/resource_metadata_storage.h"
6
7 #include "base/bind.h"
8 #include "base/file_util.h"
9 #include "base/location.h"
10 #include "base/logging.h"
11 #include "base/metrics/histogram.h"
12 #include "base/metrics/sparse_histogram.h"
13 #include "base/sequenced_task_runner.h"
14 #include "base/threading/thread_restrictions.h"
15 #include "chrome/browser/chromeos/drive/drive.pb.h"
16 #include "third_party/leveldatabase/src/include/leveldb/db.h"
17 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
18
19 namespace drive {
20 namespace internal {
21
22 namespace {
23
24 // Enum to describe DB initialization status.
25 enum DBInitStatus {
26 DB_INIT_SUCCESS,
27 DB_INIT_NOT_FOUND,
28 DB_INIT_CORRUPTION,
29 DB_INIT_IO_ERROR,
30 DB_INIT_FAILED,
31 DB_INIT_INCOMPATIBLE,
32 DB_INIT_BROKEN,
33 DB_INIT_OPENED_EXISTING_DB,
34 DB_INIT_CREATED_NEW_DB,
35 DB_INIT_REPLACED_EXISTING_DB_WITH_NEW_DB,
36 DB_INIT_MAX_VALUE,
37 };
38
39 // The name of the DB which stores the metadata.
40 const base::FilePath::CharType kResourceMapDBName[] =
41 FILE_PATH_LITERAL("resource_metadata_resource_map.db");
42
43 // The name of the DB which couldn't be opened, but is preserved just in case.
44 const base::FilePath::CharType kPreservedResourceMapDBName[] =
45 FILE_PATH_LITERAL("resource_metadata_preserved_resource_map.db");
46
47 // The name of the DB which couldn't be opened, and was replaced with a new one.
48 const base::FilePath::CharType kTrashedResourceMapDBName[] =
49 FILE_PATH_LITERAL("resource_metadata_trashed_resource_map.db");
50
51 // Meant to be a character which never happen to be in real IDs.
52 const char kDBKeyDelimeter = '\0';
53
54 // String used as a suffix of a key for a cache entry.
55 const char kCacheEntryKeySuffix[] = "CACHE";
56
57 // String used as a prefix of a key for a resource-ID-to-local-ID entry.
58 const char kIdEntryKeyPrefix[] = "ID";
59
60 // Returns a string to be used as the key for the header.
GetHeaderDBKey()61 std::string GetHeaderDBKey() {
62 std::string key;
63 key.push_back(kDBKeyDelimeter);
64 key.append("HEADER");
65 return key;
66 }
67
68 // Returns true if |key| is a key for a child entry.
IsChildEntryKey(const leveldb::Slice & key)69 bool IsChildEntryKey(const leveldb::Slice& key) {
70 return !key.empty() && key[key.size() - 1] == kDBKeyDelimeter;
71 }
72
73 // Returns a string to be used as a key for a cache entry.
GetCacheEntryKey(const std::string & id)74 std::string GetCacheEntryKey(const std::string& id) {
75 std::string key(id);
76 key.push_back(kDBKeyDelimeter);
77 key.append(kCacheEntryKeySuffix);
78 return key;
79 }
80
81 // Returns true if |key| is a key for a cache entry.
IsCacheEntryKey(const leveldb::Slice & key)82 bool IsCacheEntryKey(const leveldb::Slice& key) {
83 // A cache entry key should end with |kDBKeyDelimeter + kCacheEntryKeySuffix|.
84 const leveldb::Slice expected_suffix(kCacheEntryKeySuffix,
85 arraysize(kCacheEntryKeySuffix) - 1);
86 if (key.size() < 1 + expected_suffix.size() ||
87 key[key.size() - expected_suffix.size() - 1] != kDBKeyDelimeter)
88 return false;
89
90 const leveldb::Slice key_substring(
91 key.data() + key.size() - expected_suffix.size(), expected_suffix.size());
92 return key_substring.compare(expected_suffix) == 0;
93 }
94
95 // Returns ID extracted from a cache entry key.
GetIdFromCacheEntryKey(const leveldb::Slice & key)96 std::string GetIdFromCacheEntryKey(const leveldb::Slice& key) {
97 DCHECK(IsCacheEntryKey(key));
98 // Drop the suffix |kDBKeyDelimeter + kCacheEntryKeySuffix| from the key.
99 const size_t kSuffixLength = arraysize(kCacheEntryKeySuffix) - 1;
100 const int id_length = key.size() - 1 - kSuffixLength;
101 return std::string(key.data(), id_length);
102 }
103
104 // Returns a string to be used as a key for a resource-ID-to-local-ID entry.
GetIdEntryKey(const std::string & resource_id)105 std::string GetIdEntryKey(const std::string& resource_id) {
106 std::string key;
107 key.push_back(kDBKeyDelimeter);
108 key.append(kIdEntryKeyPrefix);
109 key.push_back(kDBKeyDelimeter);
110 key.append(resource_id);
111 return key;
112 }
113
114 // Returns true if |key| is a key for a resource-ID-to-local-ID entry.
IsIdEntryKey(const leveldb::Slice & key)115 bool IsIdEntryKey(const leveldb::Slice& key) {
116 // A resource-ID-to-local-ID entry key should start with
117 // |kDBKeyDelimeter + kIdEntryKeyPrefix + kDBKeyDelimeter|.
118 const leveldb::Slice expected_prefix(kIdEntryKeyPrefix,
119 arraysize(kIdEntryKeyPrefix) - 1);
120 if (key.size() < 2 + expected_prefix.size())
121 return false;
122 const leveldb::Slice key_substring(key.data() + 1, expected_prefix.size());
123 return key[0] == kDBKeyDelimeter &&
124 key_substring.compare(expected_prefix) == 0 &&
125 key[expected_prefix.size() + 1] == kDBKeyDelimeter;
126 }
127
128 // Converts leveldb::Status to DBInitStatus.
LevelDBStatusToDBInitStatus(const leveldb::Status status)129 DBInitStatus LevelDBStatusToDBInitStatus(const leveldb::Status status) {
130 if (status.ok())
131 return DB_INIT_SUCCESS;
132 if (status.IsNotFound())
133 return DB_INIT_NOT_FOUND;
134 if (status.IsCorruption())
135 return DB_INIT_CORRUPTION;
136 if (status.IsIOError())
137 return DB_INIT_IO_ERROR;
138 return DB_INIT_FAILED;
139 }
140
GetDefaultHeaderEntry()141 ResourceMetadataHeader GetDefaultHeaderEntry() {
142 ResourceMetadataHeader header;
143 header.set_version(ResourceMetadataStorage::kDBVersion);
144 return header;
145 }
146
MoveIfPossible(const base::FilePath & from,const base::FilePath & to)147 bool MoveIfPossible(const base::FilePath& from, const base::FilePath& to) {
148 return !base::PathExists(from) || base::Move(from, to);
149 }
150
151 } // namespace
152
Iterator(scoped_ptr<leveldb::Iterator> it)153 ResourceMetadataStorage::Iterator::Iterator(scoped_ptr<leveldb::Iterator> it)
154 : it_(it.Pass()) {
155 base::ThreadRestrictions::AssertIOAllowed();
156 DCHECK(it_);
157
158 // Skip the header entry.
159 // Note: The header entry comes before all other entries because its key
160 // starts with kDBKeyDelimeter. (i.e. '\0')
161 it_->Seek(leveldb::Slice(GetHeaderDBKey()));
162
163 Advance();
164 }
165
~Iterator()166 ResourceMetadataStorage::Iterator::~Iterator() {
167 base::ThreadRestrictions::AssertIOAllowed();
168 }
169
IsAtEnd() const170 bool ResourceMetadataStorage::Iterator::IsAtEnd() const {
171 base::ThreadRestrictions::AssertIOAllowed();
172 return !it_->Valid();
173 }
174
GetID() const175 std::string ResourceMetadataStorage::Iterator::GetID() const {
176 return it_->key().ToString();
177 }
178
GetValue() const179 const ResourceEntry& ResourceMetadataStorage::Iterator::GetValue() const {
180 base::ThreadRestrictions::AssertIOAllowed();
181 DCHECK(!IsAtEnd());
182 return entry_;
183 }
184
GetCacheEntry(FileCacheEntry * cache_entry)185 bool ResourceMetadataStorage::Iterator::GetCacheEntry(
186 FileCacheEntry* cache_entry) {
187 base::ThreadRestrictions::AssertIOAllowed();
188 DCHECK(!IsAtEnd());
189
190 // Try to seek to the cache entry.
191 std::string current_key = it_->key().ToString();
192 std::string cache_entry_key = GetCacheEntryKey(current_key);
193 it_->Seek(leveldb::Slice(cache_entry_key));
194
195 bool success = it_->Valid() &&
196 it_->key().compare(cache_entry_key) == 0 &&
197 cache_entry->ParseFromArray(it_->value().data(), it_->value().size());
198
199 // Seek back to the original position.
200 it_->Seek(leveldb::Slice(current_key));
201 DCHECK(!IsAtEnd());
202 DCHECK_EQ(current_key, it_->key().ToString());
203
204 return success;
205 }
206
Advance()207 void ResourceMetadataStorage::Iterator::Advance() {
208 base::ThreadRestrictions::AssertIOAllowed();
209 DCHECK(!IsAtEnd());
210
211 for (it_->Next() ; it_->Valid(); it_->Next()) {
212 if (!IsChildEntryKey(it_->key()) &&
213 !IsCacheEntryKey(it_->key()) &&
214 !IsIdEntryKey(it_->key()) &&
215 entry_.ParseFromArray(it_->value().data(), it_->value().size()))
216 break;
217 }
218 }
219
HasError() const220 bool ResourceMetadataStorage::Iterator::HasError() const {
221 base::ThreadRestrictions::AssertIOAllowed();
222 return !it_->status().ok();
223 }
224
CacheEntryIterator(scoped_ptr<leveldb::Iterator> it)225 ResourceMetadataStorage::CacheEntryIterator::CacheEntryIterator(
226 scoped_ptr<leveldb::Iterator> it) : it_(it.Pass()) {
227 base::ThreadRestrictions::AssertIOAllowed();
228 DCHECK(it_);
229
230 it_->SeekToFirst();
231 AdvanceInternal();
232 }
233
~CacheEntryIterator()234 ResourceMetadataStorage::CacheEntryIterator::~CacheEntryIterator() {
235 base::ThreadRestrictions::AssertIOAllowed();
236 }
237
IsAtEnd() const238 bool ResourceMetadataStorage::CacheEntryIterator::IsAtEnd() const {
239 base::ThreadRestrictions::AssertIOAllowed();
240 return !it_->Valid();
241 }
242
GetID() const243 const std::string& ResourceMetadataStorage::CacheEntryIterator::GetID() const {
244 base::ThreadRestrictions::AssertIOAllowed();
245 DCHECK(!IsAtEnd());
246 return id_;
247 }
248
249 const FileCacheEntry&
GetValue() const250 ResourceMetadataStorage::CacheEntryIterator::GetValue() const {
251 base::ThreadRestrictions::AssertIOAllowed();
252 DCHECK(!IsAtEnd());
253 return entry_;
254 }
255
Advance()256 void ResourceMetadataStorage::CacheEntryIterator::Advance() {
257 base::ThreadRestrictions::AssertIOAllowed();
258 DCHECK(!IsAtEnd());
259
260 it_->Next();
261 AdvanceInternal();
262 }
263
HasError() const264 bool ResourceMetadataStorage::CacheEntryIterator::HasError() const {
265 base::ThreadRestrictions::AssertIOAllowed();
266 return !it_->status().ok();
267 }
268
AdvanceInternal()269 void ResourceMetadataStorage::CacheEntryIterator::AdvanceInternal() {
270 for (; it_->Valid(); it_->Next()) {
271 // Skip unparsable broken entries.
272 // TODO(hashimoto): Broken entries should be cleaned up at some point.
273 if (IsCacheEntryKey(it_->key()) &&
274 entry_.ParseFromArray(it_->value().data(), it_->value().size())) {
275 id_ = GetIdFromCacheEntryKey(it_->key());
276 break;
277 }
278 }
279 }
280
281 // static
UpgradeOldDB(const base::FilePath & directory_path,const ResourceIdCanonicalizer & id_canonicalizer)282 bool ResourceMetadataStorage::UpgradeOldDB(
283 const base::FilePath& directory_path,
284 const ResourceIdCanonicalizer& id_canonicalizer) {
285 base::ThreadRestrictions::AssertIOAllowed();
286 COMPILE_ASSERT(
287 kDBVersion == 12,
288 db_version_and_this_function_should_be_updated_at_the_same_time);
289
290 const base::FilePath resource_map_path =
291 directory_path.Append(kResourceMapDBName);
292 const base::FilePath preserved_resource_map_path =
293 directory_path.Append(kPreservedResourceMapDBName);
294
295 if (base::PathExists(preserved_resource_map_path)) {
296 // Preserved DB is found. The previous attempt to create a new DB should not
297 // be successful. Discard the imperfect new DB and restore the old DB.
298 if (!base::DeleteFile(resource_map_path, false /* recursive */) ||
299 !base::Move(preserved_resource_map_path, resource_map_path))
300 return false;
301 }
302
303 if (!base::PathExists(resource_map_path))
304 return false;
305
306 // Open DB.
307 leveldb::DB* db = NULL;
308 leveldb::Options options;
309 options.max_open_files = 0; // Use minimum.
310 options.create_if_missing = false;
311 if (!leveldb::DB::Open(options, resource_map_path.AsUTF8Unsafe(), &db).ok())
312 return false;
313 scoped_ptr<leveldb::DB> resource_map(db);
314
315 // Check DB version.
316 std::string serialized_header;
317 ResourceMetadataHeader header;
318 if (!resource_map->Get(leveldb::ReadOptions(),
319 leveldb::Slice(GetHeaderDBKey()),
320 &serialized_header).ok() ||
321 !header.ParseFromString(serialized_header))
322 return false;
323 UMA_HISTOGRAM_SPARSE_SLOWLY("Drive.MetadataDBVersionBeforeUpgradeCheck",
324 header.version());
325
326 if (header.version() == kDBVersion) { // Nothing to do.
327 return true;
328 } else if (header.version() < 6) { // Too old, nothing can be done.
329 return false;
330 } else if (header.version() < 11) { // Cache entries can be reused.
331 leveldb::ReadOptions options;
332 options.verify_checksums = true;
333 scoped_ptr<leveldb::Iterator> it(resource_map->NewIterator(options));
334
335 leveldb::WriteBatch batch;
336 for (it->SeekToFirst(); it->Valid(); it->Next()) {
337 if (IsCacheEntryKey(it->key())) {
338 // The resource ID might be in old WAPI format. We need to canonicalize
339 // to the format of API service currently in use.
340 const std::string& id = GetIdFromCacheEntryKey(it->key());
341 const std::string& id_new = id_canonicalizer.Run(id);
342 if (id != id_new) {
343 batch.Delete(it->key());
344 batch.Put(GetCacheEntryKey(id_new), it->value());
345 }
346 // Before v11, resource ID was directly used as local ID. Such entries
347 // can be migrated by adding an identity ID mapping.
348 batch.Put(GetIdEntryKey(id_new), id_new);
349 } else { // Remove all entries except cache entries.
350 batch.Delete(it->key());
351 }
352 }
353 if (!it->status().ok())
354 return false;
355
356 // Put header with the latest version number.
357 std::string serialized_header;
358 if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
359 return false;
360 batch.Put(GetHeaderDBKey(), serialized_header);
361
362 return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
363 } else if (header.version() < 12) { // Cache and ID map entries are reusable.
364 leveldb::ReadOptions options;
365 options.verify_checksums = true;
366 scoped_ptr<leveldb::Iterator> it(resource_map->NewIterator(options));
367
368 leveldb::WriteBatch batch;
369 for (it->SeekToFirst(); it->Valid(); it->Next()) {
370 if (!IsCacheEntryKey(it->key()) && !IsIdEntryKey(it->key()))
371 batch.Delete(it->key());
372 }
373 if (!it->status().ok())
374 return false;
375
376 // Put header with the latest version number.
377 std::string serialized_header;
378 if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
379 return false;
380 batch.Put(GetHeaderDBKey(), serialized_header);
381
382 return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
383 }
384
385 LOG(WARNING) << "Unexpected DB version: " << header.version();
386 return false;
387 }
388
ResourceMetadataStorage(const base::FilePath & directory_path,base::SequencedTaskRunner * blocking_task_runner)389 ResourceMetadataStorage::ResourceMetadataStorage(
390 const base::FilePath& directory_path,
391 base::SequencedTaskRunner* blocking_task_runner)
392 : directory_path_(directory_path),
393 cache_file_scan_is_needed_(true),
394 blocking_task_runner_(blocking_task_runner) {
395 }
396
Destroy()397 void ResourceMetadataStorage::Destroy() {
398 blocking_task_runner_->PostTask(
399 FROM_HERE,
400 base::Bind(&ResourceMetadataStorage::DestroyOnBlockingPool,
401 base::Unretained(this)));
402 }
403
Initialize()404 bool ResourceMetadataStorage::Initialize() {
405 base::ThreadRestrictions::AssertIOAllowed();
406
407 resource_map_.reset();
408
409 const base::FilePath resource_map_path =
410 directory_path_.Append(kResourceMapDBName);
411 const base::FilePath preserved_resource_map_path =
412 directory_path_.Append(kPreservedResourceMapDBName);
413 const base::FilePath trashed_resource_map_path =
414 directory_path_.Append(kTrashedResourceMapDBName);
415
416 // Discard unneeded DBs.
417 if (!base::DeleteFile(preserved_resource_map_path, true /* recursive */) ||
418 !base::DeleteFile(trashed_resource_map_path, true /* recursive */)) {
419 LOG(ERROR) << "Failed to remove unneeded DBs.";
420 return false;
421 }
422
423 // Try to open the existing DB.
424 leveldb::DB* db = NULL;
425 leveldb::Options options;
426 options.max_open_files = 0; // Use minimum.
427 options.create_if_missing = false;
428
429 DBInitStatus open_existing_result = DB_INIT_NOT_FOUND;
430 leveldb::Status status;
431 if (base::PathExists(resource_map_path)) {
432 status = leveldb::DB::Open(options, resource_map_path.AsUTF8Unsafe(), &db);
433 open_existing_result = LevelDBStatusToDBInitStatus(status);
434 }
435
436 if (open_existing_result == DB_INIT_SUCCESS) {
437 resource_map_.reset(db);
438
439 // Check the validity of existing DB.
440 int db_version = -1;
441 ResourceMetadataHeader header;
442 if (GetHeader(&header))
443 db_version = header.version();
444
445 bool should_discard_db = true;
446 if (db_version != kDBVersion) {
447 open_existing_result = DB_INIT_INCOMPATIBLE;
448 DVLOG(1) << "Reject incompatible DB.";
449 } else if (!CheckValidity()) {
450 open_existing_result = DB_INIT_BROKEN;
451 LOG(ERROR) << "Reject invalid DB.";
452 } else {
453 should_discard_db = false;
454 }
455
456 if (should_discard_db)
457 resource_map_.reset();
458 else
459 cache_file_scan_is_needed_ = false;
460 }
461
462 UMA_HISTOGRAM_ENUMERATION("Drive.MetadataDBOpenExistingResult",
463 open_existing_result,
464 DB_INIT_MAX_VALUE);
465
466 DBInitStatus init_result = DB_INIT_OPENED_EXISTING_DB;
467
468 // Failed to open the existing DB, create new DB.
469 if (!resource_map_) {
470 // Move the existing DB to the preservation path. The moved old DB is
471 // deleted once the new DB creation succeeds, or is restored later in
472 // UpgradeOldDB() when the creation fails.
473 MoveIfPossible(resource_map_path, preserved_resource_map_path);
474
475 // Create DB.
476 options.max_open_files = 0; // Use minimum.
477 options.create_if_missing = true;
478 options.error_if_exists = true;
479
480 status = leveldb::DB::Open(options, resource_map_path.AsUTF8Unsafe(), &db);
481 if (status.ok()) {
482 resource_map_.reset(db);
483
484 if (PutHeader(GetDefaultHeaderEntry()) && // Set up header.
485 MoveIfPossible(preserved_resource_map_path, // Trash the old DB.
486 trashed_resource_map_path)) {
487 init_result = open_existing_result == DB_INIT_NOT_FOUND ?
488 DB_INIT_CREATED_NEW_DB : DB_INIT_REPLACED_EXISTING_DB_WITH_NEW_DB;
489 } else {
490 init_result = DB_INIT_FAILED;
491 resource_map_.reset();
492 }
493 } else {
494 LOG(ERROR) << "Failed to create resource map DB: " << status.ToString();
495 init_result = LevelDBStatusToDBInitStatus(status);
496 }
497 }
498
499 UMA_HISTOGRAM_ENUMERATION("Drive.MetadataDBInitResult",
500 init_result,
501 DB_INIT_MAX_VALUE);
502 return resource_map_;
503 }
504
RecoverCacheInfoFromTrashedResourceMap(RecoveredCacheInfoMap * out_info)505 void ResourceMetadataStorage::RecoverCacheInfoFromTrashedResourceMap(
506 RecoveredCacheInfoMap* out_info) {
507 const base::FilePath trashed_resource_map_path =
508 directory_path_.Append(kTrashedResourceMapDBName);
509
510 if (!base::PathExists(trashed_resource_map_path))
511 return;
512
513 leveldb::Options options;
514 options.max_open_files = 0; // Use minimum.
515 options.create_if_missing = false;
516
517 // Trashed DB may be broken, repair it first.
518 leveldb::Status status;
519 status = leveldb::RepairDB(trashed_resource_map_path.AsUTF8Unsafe(), options);
520 if (!status.ok()) {
521 LOG(ERROR) << "Failed to repair trashed DB: " << status.ToString();
522 return;
523 }
524
525 // Open it.
526 leveldb::DB* db = NULL;
527 status = leveldb::DB::Open(options, trashed_resource_map_path.AsUTF8Unsafe(),
528 &db);
529 if (!status.ok()) {
530 LOG(ERROR) << "Failed to open trashed DB: " << status.ToString();
531 return;
532 }
533 scoped_ptr<leveldb::DB> resource_map(db);
534
535 // Check DB version.
536 std::string serialized_header;
537 ResourceMetadataHeader header;
538 if (!resource_map->Get(leveldb::ReadOptions(),
539 leveldb::Slice(GetHeaderDBKey()),
540 &serialized_header).ok() ||
541 !header.ParseFromString(serialized_header) ||
542 header.version() != kDBVersion) {
543 LOG(ERROR) << "Incompatible DB version: " << header.version();
544 return;
545 }
546
547 // Collect cache entries.
548 scoped_ptr<leveldb::Iterator> it(
549 resource_map->NewIterator(leveldb::ReadOptions()));
550 for (it->SeekToFirst(); it->Valid(); it->Next()) {
551 if (IsCacheEntryKey(it->key())) {
552 const std::string& id = GetIdFromCacheEntryKey(it->key());
553 FileCacheEntry cache_entry;
554 if (cache_entry.ParseFromArray(it->value().data(), it->value().size())) {
555 RecoveredCacheInfo* info = &(*out_info)[id];
556 info->is_dirty = cache_entry.is_dirty();
557 info->md5 = cache_entry.md5();
558
559 // Get title from ResourceEntry if available.
560 std::string serialized_entry;
561 ResourceEntry entry;
562 if (resource_map->Get(leveldb::ReadOptions(),
563 leveldb::Slice(id),
564 &serialized_entry).ok() &&
565 entry.ParseFromString(serialized_entry))
566 info->title = entry.title();
567 }
568 }
569 }
570 }
571
SetLargestChangestamp(int64 largest_changestamp)572 bool ResourceMetadataStorage::SetLargestChangestamp(
573 int64 largest_changestamp) {
574 base::ThreadRestrictions::AssertIOAllowed();
575
576 ResourceMetadataHeader header;
577 if (!GetHeader(&header)) {
578 DLOG(ERROR) << "Failed to get the header.";
579 return false;
580 }
581 header.set_largest_changestamp(largest_changestamp);
582 return PutHeader(header);
583 }
584
GetLargestChangestamp()585 int64 ResourceMetadataStorage::GetLargestChangestamp() {
586 base::ThreadRestrictions::AssertIOAllowed();
587 ResourceMetadataHeader header;
588 if (!GetHeader(&header)) {
589 DLOG(ERROR) << "Failed to get the header.";
590 return 0;
591 }
592 return header.largest_changestamp();
593 }
594
PutEntry(const ResourceEntry & entry)595 bool ResourceMetadataStorage::PutEntry(const ResourceEntry& entry) {
596 base::ThreadRestrictions::AssertIOAllowed();
597
598 const std::string& id = entry.local_id();
599 DCHECK(!id.empty());
600
601 // Try to get existing entry.
602 std::string serialized_entry;
603 leveldb::Status status = resource_map_->Get(leveldb::ReadOptions(),
604 leveldb::Slice(id),
605 &serialized_entry);
606 if (!status.ok() && !status.IsNotFound()) // Unexpected errors.
607 return false;
608
609 ResourceEntry old_entry;
610 if (status.ok() && !old_entry.ParseFromString(serialized_entry))
611 return false;
612
613 // Construct write batch.
614 leveldb::WriteBatch batch;
615
616 // Remove from the old parent.
617 if (!old_entry.parent_local_id().empty()) {
618 batch.Delete(GetChildEntryKey(old_entry.parent_local_id(),
619 old_entry.base_name()));
620 }
621 // Add to the new parent.
622 if (!entry.parent_local_id().empty())
623 batch.Put(GetChildEntryKey(entry.parent_local_id(), entry.base_name()), id);
624
625 // Refresh resource-ID-to-local-ID mapping entry.
626 if (old_entry.resource_id() != entry.resource_id()) {
627 // Resource ID should not change.
628 DCHECK(old_entry.resource_id().empty() || entry.resource_id().empty());
629
630 if (!old_entry.resource_id().empty())
631 batch.Delete(GetIdEntryKey(old_entry.resource_id()));
632 if (!entry.resource_id().empty())
633 batch.Put(GetIdEntryKey(entry.resource_id()), id);
634 }
635
636 // Put the entry itself.
637 if (!entry.SerializeToString(&serialized_entry)) {
638 DLOG(ERROR) << "Failed to serialize the entry: " << id;
639 return false;
640 }
641 batch.Put(id, serialized_entry);
642
643 status = resource_map_->Write(leveldb::WriteOptions(), &batch);
644 return status.ok();
645 }
646
GetEntry(const std::string & id,ResourceEntry * out_entry)647 bool ResourceMetadataStorage::GetEntry(const std::string& id,
648 ResourceEntry* out_entry) {
649 base::ThreadRestrictions::AssertIOAllowed();
650 DCHECK(!id.empty());
651
652 std::string serialized_entry;
653 const leveldb::Status status = resource_map_->Get(leveldb::ReadOptions(),
654 leveldb::Slice(id),
655 &serialized_entry);
656 return status.ok() && out_entry->ParseFromString(serialized_entry);
657 }
658
RemoveEntry(const std::string & id)659 bool ResourceMetadataStorage::RemoveEntry(const std::string& id) {
660 base::ThreadRestrictions::AssertIOAllowed();
661 DCHECK(!id.empty());
662
663 ResourceEntry entry;
664 if (!GetEntry(id, &entry))
665 return false;
666
667 leveldb::WriteBatch batch;
668
669 // Remove from the parent.
670 if (!entry.parent_local_id().empty())
671 batch.Delete(GetChildEntryKey(entry.parent_local_id(), entry.base_name()));
672
673 // Remove resource ID-local ID mapping entry.
674 if (!entry.resource_id().empty())
675 batch.Delete(GetIdEntryKey(entry.resource_id()));
676
677 // Remove the entry itself.
678 batch.Delete(id);
679
680 const leveldb::Status status = resource_map_->Write(leveldb::WriteOptions(),
681 &batch);
682 return status.ok();
683 }
684
685 scoped_ptr<ResourceMetadataStorage::Iterator>
GetIterator()686 ResourceMetadataStorage::GetIterator() {
687 base::ThreadRestrictions::AssertIOAllowed();
688
689 scoped_ptr<leveldb::Iterator> it(
690 resource_map_->NewIterator(leveldb::ReadOptions()));
691 return make_scoped_ptr(new Iterator(it.Pass()));
692 }
693
GetChild(const std::string & parent_id,const std::string & child_name)694 std::string ResourceMetadataStorage::GetChild(const std::string& parent_id,
695 const std::string& child_name) {
696 base::ThreadRestrictions::AssertIOAllowed();
697 DCHECK(!parent_id.empty());
698 DCHECK(!child_name.empty());
699
700 std::string child_id;
701 resource_map_->Get(leveldb::ReadOptions(),
702 leveldb::Slice(GetChildEntryKey(parent_id, child_name)),
703 &child_id);
704 return child_id;
705 }
706
GetChildren(const std::string & parent_id,std::vector<std::string> * children)707 void ResourceMetadataStorage::GetChildren(const std::string& parent_id,
708 std::vector<std::string>* children) {
709 base::ThreadRestrictions::AssertIOAllowed();
710 DCHECK(!parent_id.empty());
711
712 // Iterate over all entries with keys starting with |parent_id|.
713 scoped_ptr<leveldb::Iterator> it(
714 resource_map_->NewIterator(leveldb::ReadOptions()));
715 for (it->Seek(parent_id);
716 it->Valid() && it->key().starts_with(leveldb::Slice(parent_id));
717 it->Next()) {
718 if (IsChildEntryKey(it->key()))
719 children->push_back(it->value().ToString());
720 }
721 DCHECK(it->status().ok());
722 }
723
PutCacheEntry(const std::string & id,const FileCacheEntry & entry)724 bool ResourceMetadataStorage::PutCacheEntry(const std::string& id,
725 const FileCacheEntry& entry) {
726 base::ThreadRestrictions::AssertIOAllowed();
727 DCHECK(!id.empty());
728
729 std::string serialized_entry;
730 if (!entry.SerializeToString(&serialized_entry)) {
731 DLOG(ERROR) << "Failed to serialize the entry.";
732 return false;
733 }
734
735 const leveldb::Status status = resource_map_->Put(
736 leveldb::WriteOptions(),
737 leveldb::Slice(GetCacheEntryKey(id)),
738 leveldb::Slice(serialized_entry));
739 return status.ok();
740 }
741
GetCacheEntry(const std::string & id,FileCacheEntry * out_entry)742 bool ResourceMetadataStorage::GetCacheEntry(const std::string& id,
743 FileCacheEntry* out_entry) {
744 base::ThreadRestrictions::AssertIOAllowed();
745 DCHECK(!id.empty());
746
747 std::string serialized_entry;
748 const leveldb::Status status = resource_map_->Get(
749 leveldb::ReadOptions(),
750 leveldb::Slice(GetCacheEntryKey(id)),
751 &serialized_entry);
752 return status.ok() && out_entry->ParseFromString(serialized_entry);
753 }
754
RemoveCacheEntry(const std::string & id)755 bool ResourceMetadataStorage::RemoveCacheEntry(const std::string& id) {
756 base::ThreadRestrictions::AssertIOAllowed();
757 DCHECK(!id.empty());
758
759 const leveldb::Status status = resource_map_->Delete(
760 leveldb::WriteOptions(),
761 leveldb::Slice(GetCacheEntryKey(id)));
762 return status.ok();
763 }
764
765 scoped_ptr<ResourceMetadataStorage::CacheEntryIterator>
GetCacheEntryIterator()766 ResourceMetadataStorage::GetCacheEntryIterator() {
767 base::ThreadRestrictions::AssertIOAllowed();
768
769 scoped_ptr<leveldb::Iterator> it(
770 resource_map_->NewIterator(leveldb::ReadOptions()));
771 return make_scoped_ptr(new CacheEntryIterator(it.Pass()));
772 }
773
RecoveredCacheInfo()774 ResourceMetadataStorage::RecoveredCacheInfo::RecoveredCacheInfo()
775 : is_dirty(false) {}
776
~RecoveredCacheInfo()777 ResourceMetadataStorage::RecoveredCacheInfo::~RecoveredCacheInfo() {}
778
GetIdByResourceId(const std::string & resource_id,std::string * out_id)779 bool ResourceMetadataStorage::GetIdByResourceId(
780 const std::string& resource_id,
781 std::string* out_id) {
782 base::ThreadRestrictions::AssertIOAllowed();
783 DCHECK(!resource_id.empty());
784
785 const leveldb::Status status = resource_map_->Get(
786 leveldb::ReadOptions(),
787 leveldb::Slice(GetIdEntryKey(resource_id)),
788 out_id);
789 return status.ok();
790 }
791
~ResourceMetadataStorage()792 ResourceMetadataStorage::~ResourceMetadataStorage() {
793 base::ThreadRestrictions::AssertIOAllowed();
794 }
795
DestroyOnBlockingPool()796 void ResourceMetadataStorage::DestroyOnBlockingPool() {
797 delete this;
798 }
799
800 // static
GetChildEntryKey(const std::string & parent_id,const std::string & child_name)801 std::string ResourceMetadataStorage::GetChildEntryKey(
802 const std::string& parent_id,
803 const std::string& child_name) {
804 DCHECK(!parent_id.empty());
805 DCHECK(!child_name.empty());
806
807 std::string key = parent_id;
808 key.push_back(kDBKeyDelimeter);
809 key.append(child_name);
810 key.push_back(kDBKeyDelimeter);
811 return key;
812 }
813
PutHeader(const ResourceMetadataHeader & header)814 bool ResourceMetadataStorage::PutHeader(
815 const ResourceMetadataHeader& header) {
816 base::ThreadRestrictions::AssertIOAllowed();
817
818 std::string serialized_header;
819 if (!header.SerializeToString(&serialized_header)) {
820 DLOG(ERROR) << "Failed to serialize the header";
821 return false;
822 }
823
824 const leveldb::Status status = resource_map_->Put(
825 leveldb::WriteOptions(),
826 leveldb::Slice(GetHeaderDBKey()),
827 leveldb::Slice(serialized_header));
828 return status.ok();
829 }
830
GetHeader(ResourceMetadataHeader * header)831 bool ResourceMetadataStorage::GetHeader(ResourceMetadataHeader* header) {
832 base::ThreadRestrictions::AssertIOAllowed();
833
834 std::string serialized_header;
835 const leveldb::Status status = resource_map_->Get(
836 leveldb::ReadOptions(),
837 leveldb::Slice(GetHeaderDBKey()),
838 &serialized_header);
839 return status.ok() && header->ParseFromString(serialized_header);
840 }
841
CheckValidity()842 bool ResourceMetadataStorage::CheckValidity() {
843 base::ThreadRestrictions::AssertIOAllowed();
844
845 // Perform read with checksums verification enalbed.
846 leveldb::ReadOptions options;
847 options.verify_checksums = true;
848
849 scoped_ptr<leveldb::Iterator> it(resource_map_->NewIterator(options));
850 it->SeekToFirst();
851
852 // DB is organized like this:
853 //
854 // <key> : <value>
855 // "\0HEADER" : ResourceMetadataHeader
856 // "\0ID\0|resource ID 1|" : Local ID associated to resource ID 1.
857 // "\0ID\0|resource ID 2|" : Local ID associated to resource ID 2.
858 // ...
859 // "|ID of A|" : ResourceEntry for entry A.
860 // "|ID of A|\0CACHE" : FileCacheEntry for entry A.
861 // "|ID of A|\0|child name 1|\0" : ID of the 1st child entry of entry A.
862 // "|ID of A|\0|child name 2|\0" : ID of the 2nd child entry of entry A.
863 // ...
864 // "|ID of A|\0|child name n|\0" : ID of the nth child entry of entry A.
865 // "|ID of B|" : ResourceEntry for entry B.
866 // "|ID of B|\0CACHE" : FileCacheEntry for entry B.
867 // ...
868
869 // Check the header.
870 ResourceMetadataHeader header;
871 if (!it->Valid() ||
872 it->key() != GetHeaderDBKey() || // Header entry must come first.
873 !header.ParseFromArray(it->value().data(), it->value().size()) ||
874 header.version() != kDBVersion) {
875 DLOG(ERROR) << "Invalid header detected. version = " << header.version();
876 return false;
877 }
878
879 // Check all entries.
880 size_t num_entries_with_parent = 0;
881 size_t num_child_entries = 0;
882 ResourceEntry entry;
883 std::string serialized_entry;
884 std::string child_id;
885 for (it->Next(); it->Valid(); it->Next()) {
886 // Count child entries.
887 if (IsChildEntryKey(it->key())) {
888 ++num_child_entries;
889 continue;
890 }
891
892 // Ignore cache entries.
893 if (IsCacheEntryKey(it->key()))
894 continue;
895
896 // Check if resource-ID-to-local-ID mapping is stored correctly.
897 if (IsIdEntryKey(it->key())) {
898 leveldb::Status status = resource_map_->Get(
899 options, it->value(), &serialized_entry);
900 // Resource-ID-to-local-ID mapping without entry for the local ID is ok.
901 if (status.IsNotFound())
902 continue;
903 // When the entry exists, its resource ID must be consistent.
904 const bool ok = status.ok() &&
905 entry.ParseFromString(serialized_entry) &&
906 !entry.resource_id().empty() &&
907 leveldb::Slice(GetIdEntryKey(entry.resource_id())) == it->key();
908 if (!ok) {
909 DLOG(ERROR) << "Broken ID entry. status = " << status.ToString();
910 return false;
911 }
912 continue;
913 }
914
915 // Check if stored data is broken.
916 if (!entry.ParseFromArray(it->value().data(), it->value().size())) {
917 DLOG(ERROR) << "Broken entry detected";
918 return false;
919 }
920
921 if (!entry.parent_local_id().empty()) {
922 // Check if the parent entry is stored.
923 leveldb::Status status = resource_map_->Get(
924 options,
925 leveldb::Slice(entry.parent_local_id()),
926 &serialized_entry);
927 if (!status.ok()) {
928 DLOG(ERROR) << "Can't get parent entry. status = " << status.ToString();
929 return false;
930 }
931
932 // Check if parent-child relationship is stored correctly.
933 status = resource_map_->Get(
934 options,
935 leveldb::Slice(GetChildEntryKey(entry.parent_local_id(),
936 entry.base_name())),
937 &child_id);
938 if (!status.ok() || leveldb::Slice(child_id) != it->key()) {
939 DLOG(ERROR) << "Child map is broken. status = " << status.ToString();
940 return false;
941 }
942 ++num_entries_with_parent;
943 }
944 }
945 if (!it->status().ok() || num_child_entries != num_entries_with_parent) {
946 DLOG(ERROR) << "Error during checking resource map. status = "
947 << it->status().ToString();
948 return false;
949 }
950 return true;
951 }
952
953 } // namespace internal
954 } // namespace drive
955