1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7 #include <cerrno>
8
9 #include "base/basictypes.h"
10 #include "base/files/file.h"
11 #include "base/logging.h"
12 #include "base/memory/scoped_ptr.h"
13 #include "base/metrics/histogram.h"
14 #include "base/strings/string16.h"
15 #include "base/strings/string_piece.h"
16 #include "base/strings/stringprintf.h"
17 #include "base/strings/utf_string_conversions.h"
18 #include "base/sys_info.h"
19 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
20 #include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
21 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
22 #include "third_party/leveldatabase/env_chromium.h"
23 #include "third_party/leveldatabase/env_idb.h"
24 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
25 #include "third_party/leveldatabase/src/include/leveldb/db.h"
26 #include "third_party/leveldatabase/src/include/leveldb/env.h"
27 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
28
29 using base::StringPiece;
30
31 namespace content {
32
33 // Forcing flushes to disk at the end of a transaction guarantees that the
34 // data hit disk, but drastically impacts throughput when the filesystem is
35 // busy with background compactions. Not syncing trades off reliability for
36 // performance. Note that background compactions which move data from the
37 // log to SSTs are always done with reliable writes.
38 //
39 // Sync writes are necessary on Windows for quota calculations; POSIX
40 // calculates file sizes correctly even when not synced to disk.
41 #if defined(OS_WIN)
42 static const bool kSyncWrites = true;
43 #else
44 // TODO(dgrogan): Either remove the #if block or change this back to false.
45 // See http://crbug.com/338385.
46 static const bool kSyncWrites = true;
47 #endif
48
MakeSlice(const StringPiece & s)49 static leveldb::Slice MakeSlice(const StringPiece& s) {
50 return leveldb::Slice(s.begin(), s.size());
51 }
52
MakeStringPiece(const leveldb::Slice & s)53 static StringPiece MakeStringPiece(const leveldb::Slice& s) {
54 return StringPiece(s.data(), s.size());
55 }
56
ComparatorAdapter(const LevelDBComparator * comparator)57 LevelDBDatabase::ComparatorAdapter::ComparatorAdapter(
58 const LevelDBComparator* comparator)
59 : comparator_(comparator) {}
60
Compare(const leveldb::Slice & a,const leveldb::Slice & b) const61 int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a,
62 const leveldb::Slice& b) const {
63 return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
64 }
65
Name() const66 const char* LevelDBDatabase::ComparatorAdapter::Name() const {
67 return comparator_->Name();
68 }
69
70 // TODO(jsbell): Support the methods below in the future.
FindShortestSeparator(std::string * start,const leveldb::Slice & limit) const71 void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator(
72 std::string* start,
73 const leveldb::Slice& limit) const {}
74
FindShortSuccessor(std::string * key) const75 void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor(
76 std::string* key) const {}
77
LevelDBSnapshot(LevelDBDatabase * db)78 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
79 : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
80
~LevelDBSnapshot()81 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
82
LevelDBDatabase()83 LevelDBDatabase::LevelDBDatabase() {}
84
~LevelDBDatabase()85 LevelDBDatabase::~LevelDBDatabase() {
86 // db_'s destructor uses comparator_adapter_; order of deletion is important.
87 db_.reset();
88 comparator_adapter_.reset();
89 env_.reset();
90 }
91
OpenDB(leveldb::Comparator * comparator,leveldb::Env * env,const base::FilePath & path,leveldb::DB ** db)92 static leveldb::Status OpenDB(leveldb::Comparator* comparator,
93 leveldb::Env* env,
94 const base::FilePath& path,
95 leveldb::DB** db) {
96 leveldb::Options options;
97 options.comparator = comparator;
98 options.create_if_missing = true;
99 options.paranoid_checks = true;
100 options.compression = leveldb::kSnappyCompression;
101
102 // For info about the troubles we've run into with this parameter, see:
103 // https://code.google.com/p/chromium/issues/detail?id=227313#c11
104 options.max_open_files = 80;
105 options.env = env;
106
107 // ChromiumEnv assumes UTF8, converts back to FilePath before using.
108 return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
109 }
110
Destroy(const base::FilePath & file_name)111 leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
112 leveldb::Options options;
113 options.env = leveldb::IDBEnv();
114 // ChromiumEnv assumes UTF8, converts back to FilePath before using.
115 return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
116 }
117
118 namespace {
119 class LockImpl : public LevelDBLock {
120 public:
LockImpl(leveldb::Env * env,leveldb::FileLock * lock)121 explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
122 : env_(env), lock_(lock) {}
~LockImpl()123 virtual ~LockImpl() { env_->UnlockFile(lock_); }
124 private:
125 leveldb::Env* env_;
126 leveldb::FileLock* lock_;
127
128 DISALLOW_COPY_AND_ASSIGN(LockImpl);
129 };
130 } // namespace
131
LockForTesting(const base::FilePath & file_name)132 scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
133 const base::FilePath& file_name) {
134 leveldb::Env* env = leveldb::IDBEnv();
135 base::FilePath lock_path = file_name.AppendASCII("LOCK");
136 leveldb::FileLock* lock = NULL;
137 leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
138 if (!status.ok())
139 return scoped_ptr<LevelDBLock>();
140 DCHECK(lock);
141 return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
142 }
143
CheckFreeSpace(const char * const type,const base::FilePath & file_name)144 static int CheckFreeSpace(const char* const type,
145 const base::FilePath& file_name) {
146 std::string name =
147 std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
148 int64 free_disk_space_in_k_bytes =
149 base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
150 if (free_disk_space_in_k_bytes < 0) {
151 base::Histogram::FactoryGet(
152 "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
153 1,
154 2 /*boundary*/,
155 2 /*boundary*/ + 1,
156 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
157 return -1;
158 }
159 int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
160 ? INT_MAX
161 : free_disk_space_in_k_bytes;
162 const uint64 histogram_max = static_cast<uint64>(1e9);
163 COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
164 base::Histogram::FactoryGet(name,
165 1,
166 histogram_max,
167 11 /*buckets*/,
168 base::HistogramBase::kUmaTargetedHistogramFlag)
169 ->Add(clamped_disk_space_k_bytes);
170 return clamped_disk_space_k_bytes;
171 }
172
ParseAndHistogramIOErrorDetails(const std::string & histogram_name,const leveldb::Status & s)173 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
174 const leveldb::Status& s) {
175 leveldb_env::MethodID method;
176 int error = -1;
177 leveldb_env::ErrorParsingResult result =
178 leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
179 if (result == leveldb_env::NONE)
180 return;
181 std::string method_histogram_name(histogram_name);
182 method_histogram_name.append(".EnvMethod");
183 base::LinearHistogram::FactoryGet(
184 method_histogram_name,
185 1,
186 leveldb_env::kNumEntries,
187 leveldb_env::kNumEntries + 1,
188 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
189
190 std::string error_histogram_name(histogram_name);
191
192 if (result == leveldb_env::METHOD_AND_PFE) {
193 DCHECK_LT(error, 0);
194 error_histogram_name.append(std::string(".PFE.") +
195 leveldb_env::MethodIDToString(method));
196 base::LinearHistogram::FactoryGet(
197 error_histogram_name,
198 1,
199 -base::File::FILE_ERROR_MAX,
200 -base::File::FILE_ERROR_MAX + 1,
201 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
202 } else if (result == leveldb_env::METHOD_AND_ERRNO) {
203 error_histogram_name.append(std::string(".Errno.") +
204 leveldb_env::MethodIDToString(method));
205 base::LinearHistogram::FactoryGet(
206 error_histogram_name,
207 1,
208 ERANGE + 1,
209 ERANGE + 2,
210 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
211 }
212 }
213
ParseAndHistogramCorruptionDetails(const std::string & histogram_name,const leveldb::Status & status)214 static void ParseAndHistogramCorruptionDetails(
215 const std::string& histogram_name,
216 const leveldb::Status& status) {
217 int error = leveldb_env::GetCorruptionCode(status);
218 DCHECK_GE(error, 0);
219 std::string corruption_histogram_name(histogram_name);
220 corruption_histogram_name.append(".Corruption");
221 const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
222 base::LinearHistogram::FactoryGet(
223 corruption_histogram_name,
224 1,
225 kNumPatterns,
226 kNumPatterns + 1,
227 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
228 }
229
HistogramLevelDBError(const std::string & histogram_name,const leveldb::Status & s)230 static void HistogramLevelDBError(const std::string& histogram_name,
231 const leveldb::Status& s) {
232 if (s.ok()) {
233 NOTREACHED();
234 return;
235 }
236 enum {
237 LEVEL_DB_NOT_FOUND,
238 LEVEL_DB_CORRUPTION,
239 LEVEL_DB_IO_ERROR,
240 LEVEL_DB_OTHER,
241 LEVEL_DB_MAX_ERROR
242 };
243 int leveldb_error = LEVEL_DB_OTHER;
244 if (s.IsNotFound())
245 leveldb_error = LEVEL_DB_NOT_FOUND;
246 else if (s.IsCorruption())
247 leveldb_error = LEVEL_DB_CORRUPTION;
248 else if (s.IsIOError())
249 leveldb_error = LEVEL_DB_IO_ERROR;
250 base::Histogram::FactoryGet(histogram_name,
251 1,
252 LEVEL_DB_MAX_ERROR,
253 LEVEL_DB_MAX_ERROR + 1,
254 base::HistogramBase::kUmaTargetedHistogramFlag)
255 ->Add(leveldb_error);
256 if (s.IsIOError())
257 ParseAndHistogramIOErrorDetails(histogram_name, s);
258 else
259 ParseAndHistogramCorruptionDetails(histogram_name, s);
260 }
261
Open(const base::FilePath & file_name,const LevelDBComparator * comparator,scoped_ptr<LevelDBDatabase> * result,bool * is_disk_full)262 leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
263 const LevelDBComparator* comparator,
264 scoped_ptr<LevelDBDatabase>* result,
265 bool* is_disk_full) {
266 scoped_ptr<ComparatorAdapter> comparator_adapter(
267 new ComparatorAdapter(comparator));
268
269 leveldb::DB* db;
270 const leveldb::Status s =
271 OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
272
273 if (!s.ok()) {
274 HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
275 int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
276 // Disks with <100k of free space almost never succeed in opening a
277 // leveldb database.
278 if (is_disk_full)
279 *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
280
281 LOG(ERROR) << "Failed to open LevelDB database from "
282 << file_name.AsUTF8Unsafe() << "," << s.ToString();
283 return s;
284 }
285
286 CheckFreeSpace("Success", file_name);
287
288 (*result).reset(new LevelDBDatabase);
289 (*result)->db_ = make_scoped_ptr(db);
290 (*result)->comparator_adapter_ = comparator_adapter.Pass();
291 (*result)->comparator_ = comparator;
292
293 return s;
294 }
295
OpenInMemory(const LevelDBComparator * comparator)296 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
297 const LevelDBComparator* comparator) {
298 scoped_ptr<ComparatorAdapter> comparator_adapter(
299 new ComparatorAdapter(comparator));
300 scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
301
302 leveldb::DB* db;
303 const leveldb::Status s = OpenDB(
304 comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
305
306 if (!s.ok()) {
307 LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
308 return scoped_ptr<LevelDBDatabase>();
309 }
310
311 scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
312 result->env_ = in_memory_env.Pass();
313 result->db_ = make_scoped_ptr(db);
314 result->comparator_adapter_ = comparator_adapter.Pass();
315 result->comparator_ = comparator;
316
317 return result.Pass();
318 }
319
Put(const StringPiece & key,std::string * value)320 leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
321 std::string* value) {
322 leveldb::WriteOptions write_options;
323 write_options.sync = kSyncWrites;
324
325 const leveldb::Status s =
326 db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
327 if (!s.ok())
328 LOG(ERROR) << "LevelDB put failed: " << s.ToString();
329 return s;
330 }
331
Remove(const StringPiece & key)332 leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
333 leveldb::WriteOptions write_options;
334 write_options.sync = kSyncWrites;
335
336 const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
337 if (!s.IsNotFound())
338 LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
339 return s;
340 }
341
Get(const StringPiece & key,std::string * value,bool * found,const LevelDBSnapshot * snapshot)342 leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
343 std::string* value,
344 bool* found,
345 const LevelDBSnapshot* snapshot) {
346 *found = false;
347 leveldb::ReadOptions read_options;
348 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the
349 // performance impact is too great.
350 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
351
352 const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
353 if (s.ok()) {
354 *found = true;
355 return s;
356 }
357 if (s.IsNotFound())
358 return leveldb::Status::OK();
359 HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
360 LOG(ERROR) << "LevelDB get failed: " << s.ToString();
361 return s;
362 }
363
Write(const LevelDBWriteBatch & write_batch)364 leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
365 leveldb::WriteOptions write_options;
366 write_options.sync = kSyncWrites;
367
368 const leveldb::Status s =
369 db_->Write(write_options, write_batch.write_batch_.get());
370 if (!s.ok()) {
371 HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
372 LOG(ERROR) << "LevelDB write failed: " << s.ToString();
373 }
374 return s;
375 }
376
377 namespace {
378 class IteratorImpl : public LevelDBIterator {
379 public:
~IteratorImpl()380 virtual ~IteratorImpl() {}
381
382 virtual bool IsValid() const OVERRIDE;
383 virtual leveldb::Status SeekToLast() OVERRIDE;
384 virtual leveldb::Status Seek(const StringPiece& target) OVERRIDE;
385 virtual leveldb::Status Next() OVERRIDE;
386 virtual leveldb::Status Prev() OVERRIDE;
387 virtual StringPiece Key() const OVERRIDE;
388 virtual StringPiece Value() const OVERRIDE;
389
390 private:
391 friend class content::LevelDBDatabase;
392 explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
393 void CheckStatus();
394
395 scoped_ptr<leveldb::Iterator> iterator_;
396
397 DISALLOW_COPY_AND_ASSIGN(IteratorImpl);
398 };
399 } // namespace
400
IteratorImpl(scoped_ptr<leveldb::Iterator> it)401 IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
402 : iterator_(it.Pass()) {}
403
CheckStatus()404 void IteratorImpl::CheckStatus() {
405 const leveldb::Status& s = iterator_->status();
406 if (!s.ok())
407 LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
408 }
409
IsValid() const410 bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
411
SeekToLast()412 leveldb::Status IteratorImpl::SeekToLast() {
413 iterator_->SeekToLast();
414 CheckStatus();
415 return iterator_->status();
416 }
417
Seek(const StringPiece & target)418 leveldb::Status IteratorImpl::Seek(const StringPiece& target) {
419 iterator_->Seek(MakeSlice(target));
420 CheckStatus();
421 return iterator_->status();
422 }
423
Next()424 leveldb::Status IteratorImpl::Next() {
425 DCHECK(IsValid());
426 iterator_->Next();
427 CheckStatus();
428 return iterator_->status();
429 }
430
Prev()431 leveldb::Status IteratorImpl::Prev() {
432 DCHECK(IsValid());
433 iterator_->Prev();
434 CheckStatus();
435 return iterator_->status();
436 }
437
Key() const438 StringPiece IteratorImpl::Key() const {
439 DCHECK(IsValid());
440 return MakeStringPiece(iterator_->key());
441 }
442
Value() const443 StringPiece IteratorImpl::Value() const {
444 DCHECK(IsValid());
445 return MakeStringPiece(iterator_->value());
446 }
447
CreateIterator(const LevelDBSnapshot * snapshot)448 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
449 const LevelDBSnapshot* snapshot) {
450 leveldb::ReadOptions read_options;
451 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the
452 // performance impact is too great.
453 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
454
455 scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
456 return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
457 }
458
Comparator() const459 const LevelDBComparator* LevelDBDatabase::Comparator() const {
460 return comparator_;
461 }
462
Compact(const base::StringPiece & start,const base::StringPiece & stop)463 void LevelDBDatabase::Compact(const base::StringPiece& start,
464 const base::StringPiece& stop) {
465 const leveldb::Slice start_slice = MakeSlice(start);
466 const leveldb::Slice stop_slice = MakeSlice(stop);
467 db_->CompactRange(&start_slice, &stop_slice);
468 }
469
CompactAll()470 void LevelDBDatabase::CompactAll() { db_->CompactRange(NULL, NULL); }
471
472 } // namespace content
473