1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7 #include <cerrno>
8
9 #include "base/basictypes.h"
10 #include "base/files/file.h"
11 #include "base/logging.h"
12 #include "base/memory/scoped_ptr.h"
13 #include "base/metrics/histogram.h"
14 #include "base/strings/string16.h"
15 #include "base/strings/string_piece.h"
16 #include "base/strings/stringprintf.h"
17 #include "base/strings/utf_string_conversions.h"
18 #include "base/sys_info.h"
19 #include "content/browser/indexed_db/indexed_db_class_factory.h"
20 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
21 #include "content/browser/indexed_db/leveldb/leveldb_iterator_impl.h"
22 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
23 #include "third_party/leveldatabase/env_chromium.h"
24 #include "third_party/leveldatabase/env_idb.h"
25 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
26 #include "third_party/leveldatabase/src/include/leveldb/db.h"
27 #include "third_party/leveldatabase/src/include/leveldb/env.h"
28 #include "third_party/leveldatabase/src/include/leveldb/filter_policy.h"
29 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
30
31 using base::StringPiece;
32
33 namespace content {
34
35 // Forcing flushes to disk at the end of a transaction guarantees that the
36 // data hit disk, but drastically impacts throughput when the filesystem is
37 // busy with background compactions. Not syncing trades off reliability for
38 // performance. Note that background compactions which move data from the
39 // log to SSTs are always done with reliable writes.
40 //
41 // Sync writes are necessary on Windows for quota calculations; POSIX
42 // calculates file sizes correctly even when not synced to disk.
43 #if defined(OS_WIN)
44 static const bool kSyncWrites = true;
45 #else
46 // TODO(dgrogan): Either remove the #if block or change this back to false.
47 // See http://crbug.com/338385.
48 static const bool kSyncWrites = true;
49 #endif
50
MakeSlice(const StringPiece & s)51 static leveldb::Slice MakeSlice(const StringPiece& s) {
52 return leveldb::Slice(s.begin(), s.size());
53 }
54
MakeStringPiece(const leveldb::Slice & s)55 static StringPiece MakeStringPiece(const leveldb::Slice& s) {
56 return StringPiece(s.data(), s.size());
57 }
58
ComparatorAdapter(const LevelDBComparator * comparator)59 LevelDBDatabase::ComparatorAdapter::ComparatorAdapter(
60 const LevelDBComparator* comparator)
61 : comparator_(comparator) {}
62
Compare(const leveldb::Slice & a,const leveldb::Slice & b) const63 int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a,
64 const leveldb::Slice& b) const {
65 return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
66 }
67
Name() const68 const char* LevelDBDatabase::ComparatorAdapter::Name() const {
69 return comparator_->Name();
70 }
71
72 // TODO(jsbell): Support the methods below in the future.
FindShortestSeparator(std::string * start,const leveldb::Slice & limit) const73 void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator(
74 std::string* start,
75 const leveldb::Slice& limit) const {}
76
FindShortSuccessor(std::string * key) const77 void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor(
78 std::string* key) const {}
79
LevelDBSnapshot(LevelDBDatabase * db)80 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
81 : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
82
~LevelDBSnapshot()83 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
84
LevelDBDatabase()85 LevelDBDatabase::LevelDBDatabase() {}
86
~LevelDBDatabase()87 LevelDBDatabase::~LevelDBDatabase() {
88 // db_'s destructor uses comparator_adapter_; order of deletion is important.
89 db_.reset();
90 comparator_adapter_.reset();
91 env_.reset();
92 }
93
OpenDB(leveldb::Comparator * comparator,leveldb::Env * env,const base::FilePath & path,leveldb::DB ** db,scoped_ptr<const leveldb::FilterPolicy> * filter_policy)94 static leveldb::Status OpenDB(
95 leveldb::Comparator* comparator,
96 leveldb::Env* env,
97 const base::FilePath& path,
98 leveldb::DB** db,
99 scoped_ptr<const leveldb::FilterPolicy>* filter_policy) {
100 filter_policy->reset(leveldb::NewBloomFilterPolicy(10));
101 leveldb::Options options;
102 options.comparator = comparator;
103 options.create_if_missing = true;
104 options.paranoid_checks = true;
105 options.filter_policy = filter_policy->get();
106 options.compression = leveldb::kSnappyCompression;
107
108 // For info about the troubles we've run into with this parameter, see:
109 // https://code.google.com/p/chromium/issues/detail?id=227313#c11
110 options.max_open_files = 80;
111 options.env = env;
112
113 // ChromiumEnv assumes UTF8, converts back to FilePath before using.
114 leveldb::Status s = leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
115
116 return s;
117 }
118
Destroy(const base::FilePath & file_name)119 leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
120 leveldb::Options options;
121 options.env = leveldb::IDBEnv();
122 // ChromiumEnv assumes UTF8, converts back to FilePath before using.
123 return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
124 }
125
126 namespace {
127 class LockImpl : public LevelDBLock {
128 public:
LockImpl(leveldb::Env * env,leveldb::FileLock * lock)129 explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
130 : env_(env), lock_(lock) {}
~LockImpl()131 virtual ~LockImpl() { env_->UnlockFile(lock_); }
132 private:
133 leveldb::Env* env_;
134 leveldb::FileLock* lock_;
135
136 DISALLOW_COPY_AND_ASSIGN(LockImpl);
137 };
138 } // namespace
139
LockForTesting(const base::FilePath & file_name)140 scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
141 const base::FilePath& file_name) {
142 leveldb::Env* env = leveldb::IDBEnv();
143 base::FilePath lock_path = file_name.AppendASCII("LOCK");
144 leveldb::FileLock* lock = NULL;
145 leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
146 if (!status.ok())
147 return scoped_ptr<LevelDBLock>();
148 DCHECK(lock);
149 return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
150 }
151
CheckFreeSpace(const char * const type,const base::FilePath & file_name)152 static int CheckFreeSpace(const char* const type,
153 const base::FilePath& file_name) {
154 std::string name =
155 std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
156 int64 free_disk_space_in_k_bytes =
157 base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
158 if (free_disk_space_in_k_bytes < 0) {
159 base::Histogram::FactoryGet(
160 "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
161 1,
162 2 /*boundary*/,
163 2 /*boundary*/ + 1,
164 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
165 return -1;
166 }
167 int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
168 ? INT_MAX
169 : free_disk_space_in_k_bytes;
170 const uint64 histogram_max = static_cast<uint64>(1e9);
171 COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
172 base::Histogram::FactoryGet(name,
173 1,
174 histogram_max,
175 11 /*buckets*/,
176 base::HistogramBase::kUmaTargetedHistogramFlag)
177 ->Add(clamped_disk_space_k_bytes);
178 return clamped_disk_space_k_bytes;
179 }
180
ParseAndHistogramIOErrorDetails(const std::string & histogram_name,const leveldb::Status & s)181 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
182 const leveldb::Status& s) {
183 leveldb_env::MethodID method;
184 int error = -1;
185 leveldb_env::ErrorParsingResult result =
186 leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
187 if (result == leveldb_env::NONE)
188 return;
189 std::string method_histogram_name(histogram_name);
190 method_histogram_name.append(".EnvMethod");
191 base::LinearHistogram::FactoryGet(
192 method_histogram_name,
193 1,
194 leveldb_env::kNumEntries,
195 leveldb_env::kNumEntries + 1,
196 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
197
198 std::string error_histogram_name(histogram_name);
199
200 if (result == leveldb_env::METHOD_AND_PFE) {
201 DCHECK_LT(error, 0);
202 error_histogram_name.append(std::string(".PFE.") +
203 leveldb_env::MethodIDToString(method));
204 base::LinearHistogram::FactoryGet(
205 error_histogram_name,
206 1,
207 -base::File::FILE_ERROR_MAX,
208 -base::File::FILE_ERROR_MAX + 1,
209 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
210 } else if (result == leveldb_env::METHOD_AND_ERRNO) {
211 error_histogram_name.append(std::string(".Errno.") +
212 leveldb_env::MethodIDToString(method));
213 base::LinearHistogram::FactoryGet(
214 error_histogram_name,
215 1,
216 ERANGE + 1,
217 ERANGE + 2,
218 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
219 }
220 }
221
ParseAndHistogramCorruptionDetails(const std::string & histogram_name,const leveldb::Status & status)222 static void ParseAndHistogramCorruptionDetails(
223 const std::string& histogram_name,
224 const leveldb::Status& status) {
225 int error = leveldb_env::GetCorruptionCode(status);
226 DCHECK_GE(error, 0);
227 std::string corruption_histogram_name(histogram_name);
228 corruption_histogram_name.append(".Corruption");
229 const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
230 base::LinearHistogram::FactoryGet(
231 corruption_histogram_name,
232 1,
233 kNumPatterns,
234 kNumPatterns + 1,
235 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
236 }
237
HistogramLevelDBError(const std::string & histogram_name,const leveldb::Status & s)238 static void HistogramLevelDBError(const std::string& histogram_name,
239 const leveldb::Status& s) {
240 if (s.ok()) {
241 NOTREACHED();
242 return;
243 }
244 enum {
245 LEVEL_DB_NOT_FOUND,
246 LEVEL_DB_CORRUPTION,
247 LEVEL_DB_IO_ERROR,
248 LEVEL_DB_OTHER,
249 LEVEL_DB_MAX_ERROR
250 };
251 int leveldb_error = LEVEL_DB_OTHER;
252 if (s.IsNotFound())
253 leveldb_error = LEVEL_DB_NOT_FOUND;
254 else if (s.IsCorruption())
255 leveldb_error = LEVEL_DB_CORRUPTION;
256 else if (s.IsIOError())
257 leveldb_error = LEVEL_DB_IO_ERROR;
258 base::Histogram::FactoryGet(histogram_name,
259 1,
260 LEVEL_DB_MAX_ERROR,
261 LEVEL_DB_MAX_ERROR + 1,
262 base::HistogramBase::kUmaTargetedHistogramFlag)
263 ->Add(leveldb_error);
264 if (s.IsIOError())
265 ParseAndHistogramIOErrorDetails(histogram_name, s);
266 else
267 ParseAndHistogramCorruptionDetails(histogram_name, s);
268 }
269
Open(const base::FilePath & file_name,const LevelDBComparator * comparator,scoped_ptr<LevelDBDatabase> * result,bool * is_disk_full)270 leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
271 const LevelDBComparator* comparator,
272 scoped_ptr<LevelDBDatabase>* result,
273 bool* is_disk_full) {
274 base::TimeTicks begin_time = base::TimeTicks::Now();
275
276 scoped_ptr<ComparatorAdapter> comparator_adapter(
277 new ComparatorAdapter(comparator));
278
279 leveldb::DB* db;
280 scoped_ptr<const leveldb::FilterPolicy> filter_policy;
281 const leveldb::Status s = OpenDB(comparator_adapter.get(),
282 leveldb::IDBEnv(),
283 file_name,
284 &db,
285 &filter_policy);
286
287 if (!s.ok()) {
288 HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
289 int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
290 // Disks with <100k of free space almost never succeed in opening a
291 // leveldb database.
292 if (is_disk_full)
293 *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
294
295 LOG(ERROR) << "Failed to open LevelDB database from "
296 << file_name.AsUTF8Unsafe() << "," << s.ToString();
297 return s;
298 }
299
300 UMA_HISTOGRAM_MEDIUM_TIMES("WebCore.IndexedDB.LevelDB.OpenTime",
301 base::TimeTicks::Now() - begin_time);
302
303 CheckFreeSpace("Success", file_name);
304
305 (*result).reset(new LevelDBDatabase);
306 (*result)->db_ = make_scoped_ptr(db);
307 (*result)->comparator_adapter_ = comparator_adapter.Pass();
308 (*result)->comparator_ = comparator;
309 (*result)->filter_policy_ = filter_policy.Pass();
310
311 return s;
312 }
313
OpenInMemory(const LevelDBComparator * comparator)314 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
315 const LevelDBComparator* comparator) {
316 scoped_ptr<ComparatorAdapter> comparator_adapter(
317 new ComparatorAdapter(comparator));
318 scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
319
320 leveldb::DB* db;
321 scoped_ptr<const leveldb::FilterPolicy> filter_policy;
322 const leveldb::Status s = OpenDB(comparator_adapter.get(),
323 in_memory_env.get(),
324 base::FilePath(),
325 &db,
326 &filter_policy);
327
328 if (!s.ok()) {
329 LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
330 return scoped_ptr<LevelDBDatabase>();
331 }
332
333 scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
334 result->env_ = in_memory_env.Pass();
335 result->db_ = make_scoped_ptr(db);
336 result->comparator_adapter_ = comparator_adapter.Pass();
337 result->comparator_ = comparator;
338 result->filter_policy_ = filter_policy.Pass();
339
340 return result.Pass();
341 }
342
Put(const StringPiece & key,std::string * value)343 leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
344 std::string* value) {
345 base::TimeTicks begin_time = base::TimeTicks::Now();
346
347 leveldb::WriteOptions write_options;
348 write_options.sync = kSyncWrites;
349
350 const leveldb::Status s =
351 db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
352 if (!s.ok())
353 LOG(ERROR) << "LevelDB put failed: " << s.ToString();
354 else
355 UMA_HISTOGRAM_TIMES("WebCore.IndexedDB.LevelDB.PutTime",
356 base::TimeTicks::Now() - begin_time);
357 return s;
358 }
359
Remove(const StringPiece & key)360 leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
361 leveldb::WriteOptions write_options;
362 write_options.sync = kSyncWrites;
363
364 const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
365 if (!s.IsNotFound())
366 LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
367 return s;
368 }
369
Get(const StringPiece & key,std::string * value,bool * found,const LevelDBSnapshot * snapshot)370 leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
371 std::string* value,
372 bool* found,
373 const LevelDBSnapshot* snapshot) {
374 *found = false;
375 leveldb::ReadOptions read_options;
376 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the
377 // performance impact is too great.
378 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
379
380 const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
381 if (s.ok()) {
382 *found = true;
383 return s;
384 }
385 if (s.IsNotFound())
386 return leveldb::Status::OK();
387 HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
388 LOG(ERROR) << "LevelDB get failed: " << s.ToString();
389 return s;
390 }
391
Write(const LevelDBWriteBatch & write_batch)392 leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
393 base::TimeTicks begin_time = base::TimeTicks::Now();
394 leveldb::WriteOptions write_options;
395 write_options.sync = kSyncWrites;
396
397 const leveldb::Status s =
398 db_->Write(write_options, write_batch.write_batch_.get());
399 if (!s.ok()) {
400 HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
401 LOG(ERROR) << "LevelDB write failed: " << s.ToString();
402 } else {
403 UMA_HISTOGRAM_TIMES("WebCore.IndexedDB.LevelDB.WriteTime",
404 base::TimeTicks::Now() - begin_time);
405 }
406 return s;
407 }
408
CreateIterator(const LevelDBSnapshot * snapshot)409 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
410 const LevelDBSnapshot* snapshot) {
411 leveldb::ReadOptions read_options;
412 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the
413 // performance impact is too great.
414 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
415
416 scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
417 return scoped_ptr<LevelDBIterator>(
418 IndexedDBClassFactory::Get()->CreateIteratorImpl(i.Pass()));
419 }
420
Comparator() const421 const LevelDBComparator* LevelDBDatabase::Comparator() const {
422 return comparator_;
423 }
424
Compact(const base::StringPiece & start,const base::StringPiece & stop)425 void LevelDBDatabase::Compact(const base::StringPiece& start,
426 const base::StringPiece& stop) {
427 const leveldb::Slice start_slice = MakeSlice(start);
428 const leveldb::Slice stop_slice = MakeSlice(stop);
429 // NULL batch means just wait for earlier writes to be done
430 db_->Write(leveldb::WriteOptions(), NULL);
431 db_->CompactRange(&start_slice, &stop_slice);
432 }
433
CompactAll()434 void LevelDBDatabase::CompactAll() { db_->CompactRange(NULL, NULL); }
435
436 } // namespace content
437