adding pods method of package managing
This commit is contained in:
Generated
+79
@@ -0,0 +1,79 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/builder.h"
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/table_cache.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
Status BuildTable(const std::string& dbname, Env* env, const Options& options,
|
||||
TableCache* table_cache, Iterator* iter, FileMetaData* meta) {
|
||||
Status s;
|
||||
meta->file_size = 0;
|
||||
iter->SeekToFirst();
|
||||
|
||||
std::string fname = TableFileName(dbname, meta->number);
|
||||
if (iter->Valid()) {
|
||||
WritableFile* file;
|
||||
s = env->NewWritableFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
|
||||
TableBuilder* builder = new TableBuilder(options, file);
|
||||
meta->smallest.DecodeFrom(iter->key());
|
||||
for (; iter->Valid(); iter->Next()) {
|
||||
Slice key = iter->key();
|
||||
meta->largest.DecodeFrom(key);
|
||||
builder->Add(key, iter->value());
|
||||
}
|
||||
|
||||
// Finish and check for builder errors
|
||||
s = builder->Finish();
|
||||
if (s.ok()) {
|
||||
meta->file_size = builder->FileSize();
|
||||
assert(meta->file_size > 0);
|
||||
}
|
||||
delete builder;
|
||||
|
||||
// Finish and check for file errors
|
||||
if (s.ok()) {
|
||||
s = file->Sync();
|
||||
}
|
||||
if (s.ok()) {
|
||||
s = file->Close();
|
||||
}
|
||||
delete file;
|
||||
file = nullptr;
|
||||
|
||||
if (s.ok()) {
|
||||
// Verify that the table is usable
|
||||
Iterator* it = table_cache->NewIterator(ReadOptions(), meta->number,
|
||||
meta->file_size);
|
||||
s = it->status();
|
||||
delete it;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for input iterator errors
|
||||
if (!iter->status().ok()) {
|
||||
s = iter->status();
|
||||
}
|
||||
|
||||
if (s.ok() && meta->file_size > 0) {
|
||||
// Keep it
|
||||
} else {
|
||||
env->DeleteFile(fname);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
Generated
+30
@@ -0,0 +1,30 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_BUILDER_H_
|
||||
#define STORAGE_LEVELDB_DB_BUILDER_H_
|
||||
|
||||
#include "leveldb/status.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
struct Options;
|
||||
struct FileMetaData;
|
||||
|
||||
class Env;
|
||||
class Iterator;
|
||||
class TableCache;
|
||||
class VersionEdit;
|
||||
|
||||
// Build a Table file from the contents of *iter. The generated file
|
||||
// will be named according to meta->number. On success, the rest of
|
||||
// *meta will be filled with metadata about the generated table.
|
||||
// If no data is present in *iter, meta->file_size will be set to
|
||||
// zero, and no Table file will be produced.
|
||||
Status BuildTable(const std::string& dbname, Env* env, const Options& options,
|
||||
TableCache* table_cache, Iterator* iter, FileMetaData* meta);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_BUILDER_H_
|
||||
Generated
+566
@@ -0,0 +1,566 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "leveldb/c.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/filter_policy.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
|
||||
using leveldb::Cache;
|
||||
using leveldb::Comparator;
|
||||
using leveldb::CompressionType;
|
||||
using leveldb::DB;
|
||||
using leveldb::Env;
|
||||
using leveldb::FileLock;
|
||||
using leveldb::FilterPolicy;
|
||||
using leveldb::Iterator;
|
||||
using leveldb::kMajorVersion;
|
||||
using leveldb::kMinorVersion;
|
||||
using leveldb::Logger;
|
||||
using leveldb::NewBloomFilterPolicy;
|
||||
using leveldb::NewLRUCache;
|
||||
using leveldb::Options;
|
||||
using leveldb::RandomAccessFile;
|
||||
using leveldb::Range;
|
||||
using leveldb::ReadOptions;
|
||||
using leveldb::SequentialFile;
|
||||
using leveldb::Slice;
|
||||
using leveldb::Snapshot;
|
||||
using leveldb::Status;
|
||||
using leveldb::WritableFile;
|
||||
using leveldb::WriteBatch;
|
||||
using leveldb::WriteOptions;
|
||||
|
||||
extern "C" {
|
||||
|
||||
struct leveldb_t {
|
||||
DB* rep;
|
||||
};
|
||||
struct leveldb_iterator_t {
|
||||
Iterator* rep;
|
||||
};
|
||||
struct leveldb_writebatch_t {
|
||||
WriteBatch rep;
|
||||
};
|
||||
struct leveldb_snapshot_t {
|
||||
const Snapshot* rep;
|
||||
};
|
||||
struct leveldb_readoptions_t {
|
||||
ReadOptions rep;
|
||||
};
|
||||
struct leveldb_writeoptions_t {
|
||||
WriteOptions rep;
|
||||
};
|
||||
struct leveldb_options_t {
|
||||
Options rep;
|
||||
};
|
||||
struct leveldb_cache_t {
|
||||
Cache* rep;
|
||||
};
|
||||
struct leveldb_seqfile_t {
|
||||
SequentialFile* rep;
|
||||
};
|
||||
struct leveldb_randomfile_t {
|
||||
RandomAccessFile* rep;
|
||||
};
|
||||
struct leveldb_writablefile_t {
|
||||
WritableFile* rep;
|
||||
};
|
||||
struct leveldb_logger_t {
|
||||
Logger* rep;
|
||||
};
|
||||
struct leveldb_filelock_t {
|
||||
FileLock* rep;
|
||||
};
|
||||
|
||||
struct leveldb_comparator_t : public Comparator {
|
||||
virtual ~leveldb_comparator_t() { (*destructor_)(state_); }
|
||||
|
||||
virtual int Compare(const Slice& a, const Slice& b) const {
|
||||
return (*compare_)(state_, a.data(), a.size(), b.data(), b.size());
|
||||
}
|
||||
|
||||
virtual const char* Name() const { return (*name_)(state_); }
|
||||
|
||||
// No-ops since the C binding does not support key shortening methods.
|
||||
virtual void FindShortestSeparator(std::string*, const Slice&) const {}
|
||||
virtual void FindShortSuccessor(std::string* key) const {}
|
||||
|
||||
void* state_;
|
||||
void (*destructor_)(void*);
|
||||
int (*compare_)(void*, const char* a, size_t alen, const char* b,
|
||||
size_t blen);
|
||||
const char* (*name_)(void*);
|
||||
};
|
||||
|
||||
struct leveldb_filterpolicy_t : public FilterPolicy {
|
||||
virtual ~leveldb_filterpolicy_t() { (*destructor_)(state_); }
|
||||
|
||||
virtual const char* Name() const { return (*name_)(state_); }
|
||||
|
||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
|
||||
std::vector<const char*> key_pointers(n);
|
||||
std::vector<size_t> key_sizes(n);
|
||||
for (int i = 0; i < n; i++) {
|
||||
key_pointers[i] = keys[i].data();
|
||||
key_sizes[i] = keys[i].size();
|
||||
}
|
||||
size_t len;
|
||||
char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len);
|
||||
dst->append(filter, len);
|
||||
free(filter);
|
||||
}
|
||||
|
||||
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const {
|
||||
return (*key_match_)(state_, key.data(), key.size(), filter.data(),
|
||||
filter.size());
|
||||
}
|
||||
|
||||
void* state_;
|
||||
void (*destructor_)(void*);
|
||||
const char* (*name_)(void*);
|
||||
char* (*create_)(void*, const char* const* key_array,
|
||||
const size_t* key_length_array, int num_keys,
|
||||
size_t* filter_length);
|
||||
unsigned char (*key_match_)(void*, const char* key, size_t length,
|
||||
const char* filter, size_t filter_length);
|
||||
};
|
||||
|
||||
struct leveldb_env_t {
|
||||
Env* rep;
|
||||
bool is_default;
|
||||
};
|
||||
|
||||
static bool SaveError(char** errptr, const Status& s) {
|
||||
assert(errptr != nullptr);
|
||||
if (s.ok()) {
|
||||
return false;
|
||||
} else if (*errptr == nullptr) {
|
||||
*errptr = strdup(s.ToString().c_str());
|
||||
} else {
|
||||
// TODO(sanjay): Merge with existing error?
|
||||
free(*errptr);
|
||||
*errptr = strdup(s.ToString().c_str());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static char* CopyString(const std::string& str) {
|
||||
char* result = reinterpret_cast<char*>(malloc(sizeof(char) * str.size()));
|
||||
memcpy(result, str.data(), sizeof(char) * str.size());
|
||||
return result;
|
||||
}
|
||||
|
||||
leveldb_t* leveldb_open(const leveldb_options_t* options, const char* name,
|
||||
char** errptr) {
|
||||
DB* db;
|
||||
if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) {
|
||||
return nullptr;
|
||||
}
|
||||
leveldb_t* result = new leveldb_t;
|
||||
result->rep = db;
|
||||
return result;
|
||||
}
|
||||
|
||||
void leveldb_close(leveldb_t* db) {
|
||||
delete db->rep;
|
||||
delete db;
|
||||
}
|
||||
|
||||
void leveldb_put(leveldb_t* db, const leveldb_writeoptions_t* options,
|
||||
const char* key, size_t keylen, const char* val, size_t vallen,
|
||||
char** errptr) {
|
||||
SaveError(errptr,
|
||||
db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen)));
|
||||
}
|
||||
|
||||
void leveldb_delete(leveldb_t* db, const leveldb_writeoptions_t* options,
|
||||
const char* key, size_t keylen, char** errptr) {
|
||||
SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen)));
|
||||
}
|
||||
|
||||
void leveldb_write(leveldb_t* db, const leveldb_writeoptions_t* options,
|
||||
leveldb_writebatch_t* batch, char** errptr) {
|
||||
SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
|
||||
}
|
||||
|
||||
char* leveldb_get(leveldb_t* db, const leveldb_readoptions_t* options,
|
||||
const char* key, size_t keylen, size_t* vallen,
|
||||
char** errptr) {
|
||||
char* result = nullptr;
|
||||
std::string tmp;
|
||||
Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp);
|
||||
if (s.ok()) {
|
||||
*vallen = tmp.size();
|
||||
result = CopyString(tmp);
|
||||
} else {
|
||||
*vallen = 0;
|
||||
if (!s.IsNotFound()) {
|
||||
SaveError(errptr, s);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
leveldb_iterator_t* leveldb_create_iterator(
|
||||
leveldb_t* db, const leveldb_readoptions_t* options) {
|
||||
leveldb_iterator_t* result = new leveldb_iterator_t;
|
||||
result->rep = db->rep->NewIterator(options->rep);
|
||||
return result;
|
||||
}
|
||||
|
||||
const leveldb_snapshot_t* leveldb_create_snapshot(leveldb_t* db) {
|
||||
leveldb_snapshot_t* result = new leveldb_snapshot_t;
|
||||
result->rep = db->rep->GetSnapshot();
|
||||
return result;
|
||||
}
|
||||
|
||||
void leveldb_release_snapshot(leveldb_t* db,
|
||||
const leveldb_snapshot_t* snapshot) {
|
||||
db->rep->ReleaseSnapshot(snapshot->rep);
|
||||
delete snapshot;
|
||||
}
|
||||
|
||||
char* leveldb_property_value(leveldb_t* db, const char* propname) {
|
||||
std::string tmp;
|
||||
if (db->rep->GetProperty(Slice(propname), &tmp)) {
|
||||
// We use strdup() since we expect human readable output.
|
||||
return strdup(tmp.c_str());
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
void leveldb_approximate_sizes(leveldb_t* db, int num_ranges,
|
||||
const char* const* range_start_key,
|
||||
const size_t* range_start_key_len,
|
||||
const char* const* range_limit_key,
|
||||
const size_t* range_limit_key_len,
|
||||
uint64_t* sizes) {
|
||||
Range* ranges = new Range[num_ranges];
|
||||
for (int i = 0; i < num_ranges; i++) {
|
||||
ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]);
|
||||
ranges[i].limit = Slice(range_limit_key[i], range_limit_key_len[i]);
|
||||
}
|
||||
db->rep->GetApproximateSizes(ranges, num_ranges, sizes);
|
||||
delete[] ranges;
|
||||
}
|
||||
|
||||
void leveldb_compact_range(leveldb_t* db, const char* start_key,
|
||||
size_t start_key_len, const char* limit_key,
|
||||
size_t limit_key_len) {
|
||||
Slice a, b;
|
||||
db->rep->CompactRange(
|
||||
// Pass null Slice if corresponding "const char*" is null
|
||||
(start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr),
|
||||
(limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr));
|
||||
}
|
||||
|
||||
void leveldb_destroy_db(const leveldb_options_t* options, const char* name,
|
||||
char** errptr) {
|
||||
SaveError(errptr, DestroyDB(name, options->rep));
|
||||
}
|
||||
|
||||
void leveldb_repair_db(const leveldb_options_t* options, const char* name,
|
||||
char** errptr) {
|
||||
SaveError(errptr, RepairDB(name, options->rep));
|
||||
}
|
||||
|
||||
void leveldb_iter_destroy(leveldb_iterator_t* iter) {
|
||||
delete iter->rep;
|
||||
delete iter;
|
||||
}
|
||||
|
||||
unsigned char leveldb_iter_valid(const leveldb_iterator_t* iter) {
|
||||
return iter->rep->Valid();
|
||||
}
|
||||
|
||||
void leveldb_iter_seek_to_first(leveldb_iterator_t* iter) {
|
||||
iter->rep->SeekToFirst();
|
||||
}
|
||||
|
||||
void leveldb_iter_seek_to_last(leveldb_iterator_t* iter) {
|
||||
iter->rep->SeekToLast();
|
||||
}
|
||||
|
||||
void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) {
|
||||
iter->rep->Seek(Slice(k, klen));
|
||||
}
|
||||
|
||||
void leveldb_iter_next(leveldb_iterator_t* iter) { iter->rep->Next(); }
|
||||
|
||||
void leveldb_iter_prev(leveldb_iterator_t* iter) { iter->rep->Prev(); }
|
||||
|
||||
const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) {
|
||||
Slice s = iter->rep->key();
|
||||
*klen = s.size();
|
||||
return s.data();
|
||||
}
|
||||
|
||||
const char* leveldb_iter_value(const leveldb_iterator_t* iter, size_t* vlen) {
|
||||
Slice s = iter->rep->value();
|
||||
*vlen = s.size();
|
||||
return s.data();
|
||||
}
|
||||
|
||||
void leveldb_iter_get_error(const leveldb_iterator_t* iter, char** errptr) {
|
||||
SaveError(errptr, iter->rep->status());
|
||||
}
|
||||
|
||||
leveldb_writebatch_t* leveldb_writebatch_create() {
|
||||
return new leveldb_writebatch_t;
|
||||
}
|
||||
|
||||
void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { delete b; }
|
||||
|
||||
void leveldb_writebatch_clear(leveldb_writebatch_t* b) { b->rep.Clear(); }
|
||||
|
||||
void leveldb_writebatch_put(leveldb_writebatch_t* b, const char* key,
|
||||
size_t klen, const char* val, size_t vlen) {
|
||||
b->rep.Put(Slice(key, klen), Slice(val, vlen));
|
||||
}
|
||||
|
||||
void leveldb_writebatch_delete(leveldb_writebatch_t* b, const char* key,
|
||||
size_t klen) {
|
||||
b->rep.Delete(Slice(key, klen));
|
||||
}
|
||||
|
||||
void leveldb_writebatch_iterate(const leveldb_writebatch_t* b, void* state,
|
||||
void (*put)(void*, const char* k, size_t klen,
|
||||
const char* v, size_t vlen),
|
||||
void (*deleted)(void*, const char* k,
|
||||
size_t klen)) {
|
||||
class H : public WriteBatch::Handler {
|
||||
public:
|
||||
void* state_;
|
||||
void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
|
||||
void (*deleted_)(void*, const char* k, size_t klen);
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
(*put_)(state_, key.data(), key.size(), value.data(), value.size());
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
(*deleted_)(state_, key.data(), key.size());
|
||||
}
|
||||
};
|
||||
H handler;
|
||||
handler.state_ = state;
|
||||
handler.put_ = put;
|
||||
handler.deleted_ = deleted;
|
||||
b->rep.Iterate(&handler);
|
||||
}
|
||||
|
||||
void leveldb_writebatch_append(leveldb_writebatch_t* destination,
|
||||
const leveldb_writebatch_t* source) {
|
||||
destination->rep.Append(source->rep);
|
||||
}
|
||||
|
||||
leveldb_options_t* leveldb_options_create() { return new leveldb_options_t; }
|
||||
|
||||
void leveldb_options_destroy(leveldb_options_t* options) { delete options; }
|
||||
|
||||
void leveldb_options_set_comparator(leveldb_options_t* opt,
|
||||
leveldb_comparator_t* cmp) {
|
||||
opt->rep.comparator = cmp;
|
||||
}
|
||||
|
||||
void leveldb_options_set_filter_policy(leveldb_options_t* opt,
|
||||
leveldb_filterpolicy_t* policy) {
|
||||
opt->rep.filter_policy = policy;
|
||||
}
|
||||
|
||||
void leveldb_options_set_create_if_missing(leveldb_options_t* opt,
|
||||
unsigned char v) {
|
||||
opt->rep.create_if_missing = v;
|
||||
}
|
||||
|
||||
void leveldb_options_set_error_if_exists(leveldb_options_t* opt,
|
||||
unsigned char v) {
|
||||
opt->rep.error_if_exists = v;
|
||||
}
|
||||
|
||||
void leveldb_options_set_paranoid_checks(leveldb_options_t* opt,
|
||||
unsigned char v) {
|
||||
opt->rep.paranoid_checks = v;
|
||||
}
|
||||
|
||||
void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) {
|
||||
opt->rep.env = (env ? env->rep : nullptr);
|
||||
}
|
||||
|
||||
void leveldb_options_set_info_log(leveldb_options_t* opt, leveldb_logger_t* l) {
|
||||
opt->rep.info_log = (l ? l->rep : nullptr);
|
||||
}
|
||||
|
||||
void leveldb_options_set_write_buffer_size(leveldb_options_t* opt, size_t s) {
|
||||
opt->rep.write_buffer_size = s;
|
||||
}
|
||||
|
||||
void leveldb_options_set_max_open_files(leveldb_options_t* opt, int n) {
|
||||
opt->rep.max_open_files = n;
|
||||
}
|
||||
|
||||
void leveldb_options_set_cache(leveldb_options_t* opt, leveldb_cache_t* c) {
|
||||
opt->rep.block_cache = c->rep;
|
||||
}
|
||||
|
||||
void leveldb_options_set_block_size(leveldb_options_t* opt, size_t s) {
|
||||
opt->rep.block_size = s;
|
||||
}
|
||||
|
||||
void leveldb_options_set_block_restart_interval(leveldb_options_t* opt, int n) {
|
||||
opt->rep.block_restart_interval = n;
|
||||
}
|
||||
|
||||
void leveldb_options_set_max_file_size(leveldb_options_t* opt, size_t s) {
|
||||
opt->rep.max_file_size = s;
|
||||
}
|
||||
|
||||
void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
|
||||
opt->rep.compression = static_cast<CompressionType>(t);
|
||||
}
|
||||
|
||||
leveldb_comparator_t* leveldb_comparator_create(
|
||||
void* state, void (*destructor)(void*),
|
||||
int (*compare)(void*, const char* a, size_t alen, const char* b,
|
||||
size_t blen),
|
||||
const char* (*name)(void*)) {
|
||||
leveldb_comparator_t* result = new leveldb_comparator_t;
|
||||
result->state_ = state;
|
||||
result->destructor_ = destructor;
|
||||
result->compare_ = compare;
|
||||
result->name_ = name;
|
||||
return result;
|
||||
}
|
||||
|
||||
void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { delete cmp; }
|
||||
|
||||
leveldb_filterpolicy_t* leveldb_filterpolicy_create(
|
||||
void* state, void (*destructor)(void*),
|
||||
char* (*create_filter)(void*, const char* const* key_array,
|
||||
const size_t* key_length_array, int num_keys,
|
||||
size_t* filter_length),
|
||||
unsigned char (*key_may_match)(void*, const char* key, size_t length,
|
||||
const char* filter, size_t filter_length),
|
||||
const char* (*name)(void*)) {
|
||||
leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t;
|
||||
result->state_ = state;
|
||||
result->destructor_ = destructor;
|
||||
result->create_ = create_filter;
|
||||
result->key_match_ = key_may_match;
|
||||
result->name_ = name;
|
||||
return result;
|
||||
}
|
||||
|
||||
void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t* filter) {
|
||||
delete filter;
|
||||
}
|
||||
|
||||
leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) {
|
||||
// Make a leveldb_filterpolicy_t, but override all of its methods so
|
||||
// they delegate to a NewBloomFilterPolicy() instead of user
|
||||
// supplied C functions.
|
||||
struct Wrapper : public leveldb_filterpolicy_t {
|
||||
static void DoNothing(void*) {}
|
||||
|
||||
~Wrapper() { delete rep_; }
|
||||
const char* Name() const { return rep_->Name(); }
|
||||
void CreateFilter(const Slice* keys, int n, std::string* dst) const {
|
||||
return rep_->CreateFilter(keys, n, dst);
|
||||
}
|
||||
bool KeyMayMatch(const Slice& key, const Slice& filter) const {
|
||||
return rep_->KeyMayMatch(key, filter);
|
||||
}
|
||||
|
||||
const FilterPolicy* rep_;
|
||||
};
|
||||
Wrapper* wrapper = new Wrapper;
|
||||
wrapper->rep_ = NewBloomFilterPolicy(bits_per_key);
|
||||
wrapper->state_ = nullptr;
|
||||
wrapper->destructor_ = &Wrapper::DoNothing;
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
leveldb_readoptions_t* leveldb_readoptions_create() {
|
||||
return new leveldb_readoptions_t;
|
||||
}
|
||||
|
||||
void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { delete opt; }
|
||||
|
||||
void leveldb_readoptions_set_verify_checksums(leveldb_readoptions_t* opt,
|
||||
unsigned char v) {
|
||||
opt->rep.verify_checksums = v;
|
||||
}
|
||||
|
||||
void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t* opt,
|
||||
unsigned char v) {
|
||||
opt->rep.fill_cache = v;
|
||||
}
|
||||
|
||||
void leveldb_readoptions_set_snapshot(leveldb_readoptions_t* opt,
|
||||
const leveldb_snapshot_t* snap) {
|
||||
opt->rep.snapshot = (snap ? snap->rep : nullptr);
|
||||
}
|
||||
|
||||
leveldb_writeoptions_t* leveldb_writeoptions_create() {
|
||||
return new leveldb_writeoptions_t;
|
||||
}
|
||||
|
||||
void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { delete opt; }
|
||||
|
||||
void leveldb_writeoptions_set_sync(leveldb_writeoptions_t* opt,
|
||||
unsigned char v) {
|
||||
opt->rep.sync = v;
|
||||
}
|
||||
|
||||
leveldb_cache_t* leveldb_cache_create_lru(size_t capacity) {
|
||||
leveldb_cache_t* c = new leveldb_cache_t;
|
||||
c->rep = NewLRUCache(capacity);
|
||||
return c;
|
||||
}
|
||||
|
||||
void leveldb_cache_destroy(leveldb_cache_t* cache) {
|
||||
delete cache->rep;
|
||||
delete cache;
|
||||
}
|
||||
|
||||
leveldb_env_t* leveldb_create_default_env() {
|
||||
leveldb_env_t* result = new leveldb_env_t;
|
||||
result->rep = Env::Default();
|
||||
result->is_default = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
void leveldb_env_destroy(leveldb_env_t* env) {
|
||||
if (!env->is_default) delete env->rep;
|
||||
delete env;
|
||||
}
|
||||
|
||||
char* leveldb_env_get_test_directory(leveldb_env_t* env) {
|
||||
std::string result;
|
||||
if (!env->rep->GetTestDirectory(&result).ok()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
char* buffer = static_cast<char*>(malloc(result.size() + 1));
|
||||
memcpy(buffer, result.data(), result.size());
|
||||
buffer[result.size()] = '\0';
|
||||
return buffer;
|
||||
}
|
||||
|
||||
void leveldb_free(void* ptr) { free(ptr); }
|
||||
|
||||
int leveldb_major_version() { return kMajorVersion; }
|
||||
|
||||
int leveldb_minor_version() { return kMinorVersion; }
|
||||
|
||||
} // end extern "C"
|
||||
Generated
+1550
File diff suppressed because it is too large
Load Diff
Generated
+216
@@ -0,0 +1,216 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_
|
||||
#define STORAGE_LEVELDB_DB_DB_IMPL_H_
|
||||
|
||||
#include <atomic>
|
||||
#include <deque>
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/log_writer.h"
|
||||
#include "db/snapshot.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "port/port.h"
|
||||
#include "port/thread_annotations.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class MemTable;
|
||||
class TableCache;
|
||||
class Version;
|
||||
class VersionEdit;
|
||||
class VersionSet;
|
||||
|
||||
class DBImpl : public DB {
|
||||
public:
|
||||
DBImpl(const Options& options, const std::string& dbname);
|
||||
|
||||
DBImpl(const DBImpl&) = delete;
|
||||
DBImpl& operator=(const DBImpl&) = delete;
|
||||
|
||||
virtual ~DBImpl();
|
||||
|
||||
// Implementations of the DB interface
|
||||
virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value);
|
||||
virtual Status Delete(const WriteOptions&, const Slice& key);
|
||||
virtual Status Write(const WriteOptions& options, WriteBatch* updates);
|
||||
virtual Status Get(const ReadOptions& options, const Slice& key,
|
||||
std::string* value);
|
||||
virtual Iterator* NewIterator(const ReadOptions&);
|
||||
virtual const Snapshot* GetSnapshot();
|
||||
virtual void ReleaseSnapshot(const Snapshot* snapshot);
|
||||
virtual bool GetProperty(const Slice& property, std::string* value);
|
||||
virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
|
||||
virtual void CompactRange(const Slice* begin, const Slice* end);
|
||||
|
||||
// Extra methods (for testing) that are not in the public DB interface
|
||||
|
||||
// Compact any files in the named level that overlap [*begin,*end]
|
||||
void TEST_CompactRange(int level, const Slice* begin, const Slice* end);
|
||||
|
||||
// Force current memtable contents to be compacted.
|
||||
Status TEST_CompactMemTable();
|
||||
|
||||
// Return an internal iterator over the current state of the database.
|
||||
// The keys of this iterator are internal keys (see format.h).
|
||||
// The returned iterator should be deleted when no longer needed.
|
||||
Iterator* TEST_NewInternalIterator();
|
||||
|
||||
// Return the maximum overlapping data (in bytes) at next level for any
|
||||
// file at a level >= 1.
|
||||
int64_t TEST_MaxNextLevelOverlappingBytes();
|
||||
|
||||
// Record a sample of bytes read at the specified internal key.
|
||||
// Samples are taken approximately once every config::kReadBytesPeriod
|
||||
// bytes.
|
||||
void RecordReadSample(Slice key);
|
||||
|
||||
private:
|
||||
friend class DB;
|
||||
struct CompactionState;
|
||||
struct Writer;
|
||||
|
||||
// Information for a manual compaction
|
||||
struct ManualCompaction {
|
||||
int level;
|
||||
bool done;
|
||||
const InternalKey* begin; // null means beginning of key range
|
||||
const InternalKey* end; // null means end of key range
|
||||
InternalKey tmp_storage; // Used to keep track of compaction progress
|
||||
};
|
||||
|
||||
// Per level compaction stats. stats_[level] stores the stats for
|
||||
// compactions that produced data for the specified "level".
|
||||
struct CompactionStats {
|
||||
CompactionStats() : micros(0), bytes_read(0), bytes_written(0) {}
|
||||
|
||||
void Add(const CompactionStats& c) {
|
||||
this->micros += c.micros;
|
||||
this->bytes_read += c.bytes_read;
|
||||
this->bytes_written += c.bytes_written;
|
||||
}
|
||||
|
||||
int64_t micros;
|
||||
int64_t bytes_read;
|
||||
int64_t bytes_written;
|
||||
};
|
||||
|
||||
Iterator* NewInternalIterator(const ReadOptions&,
|
||||
SequenceNumber* latest_snapshot,
|
||||
uint32_t* seed);
|
||||
|
||||
Status NewDB();
|
||||
|
||||
// Recover the descriptor from persistent storage. May do a significant
|
||||
// amount of work to recover recently logged updates. Any changes to
|
||||
// be made to the descriptor are added to *edit.
|
||||
Status Recover(VersionEdit* edit, bool* save_manifest)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
void MaybeIgnoreError(Status* s) const;
|
||||
|
||||
// Delete any unneeded files and stale in-memory entries.
|
||||
void DeleteObsoleteFiles() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
// Compact the in-memory write buffer to disk. Switches to a new
|
||||
// log-file/memtable and writes a new descriptor iff successful.
|
||||
// Errors are recorded in bg_error_.
|
||||
void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
Status RecoverLogFile(uint64_t log_number, bool last_log, bool* save_manifest,
|
||||
VersionEdit* edit, SequenceNumber* max_sequence)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
Status MakeRoomForWrite(bool force /* compact even if there is room? */)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
WriteBatch* BuildBatchGroup(Writer** last_writer)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
void RecordBackgroundError(const Status& s);
|
||||
|
||||
void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
static void BGWork(void* db);
|
||||
void BackgroundCall();
|
||||
void BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
void CleanupCompaction(CompactionState* compact)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
Status DoCompactionWork(CompactionState* compact)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
Status OpenCompactionOutputFile(CompactionState* compact);
|
||||
Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input);
|
||||
Status InstallCompactionResults(CompactionState* compact)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
const Comparator* user_comparator() const {
|
||||
return internal_comparator_.user_comparator();
|
||||
}
|
||||
|
||||
// Constant after construction
|
||||
Env* const env_;
|
||||
const InternalKeyComparator internal_comparator_;
|
||||
const InternalFilterPolicy internal_filter_policy_;
|
||||
const Options options_; // options_.comparator == &internal_comparator_
|
||||
const bool owns_info_log_;
|
||||
const bool owns_cache_;
|
||||
const std::string dbname_;
|
||||
|
||||
// table_cache_ provides its own synchronization
|
||||
TableCache* const table_cache_;
|
||||
|
||||
// Lock over the persistent DB state. Non-null iff successfully acquired.
|
||||
FileLock* db_lock_;
|
||||
|
||||
// State below is protected by mutex_
|
||||
port::Mutex mutex_;
|
||||
std::atomic<bool> shutting_down_;
|
||||
port::CondVar background_work_finished_signal_ GUARDED_BY(mutex_);
|
||||
MemTable* mem_;
|
||||
MemTable* imm_ GUARDED_BY(mutex_); // Memtable being compacted
|
||||
std::atomic<bool> has_imm_; // So bg thread can detect non-null imm_
|
||||
WritableFile* logfile_;
|
||||
uint64_t logfile_number_ GUARDED_BY(mutex_);
|
||||
log::Writer* log_;
|
||||
uint32_t seed_ GUARDED_BY(mutex_); // For sampling.
|
||||
|
||||
// Queue of writers.
|
||||
std::deque<Writer*> writers_ GUARDED_BY(mutex_);
|
||||
WriteBatch* tmp_batch_ GUARDED_BY(mutex_);
|
||||
|
||||
SnapshotList snapshots_ GUARDED_BY(mutex_);
|
||||
|
||||
// Set of table files to protect from deletion because they are
|
||||
// part of ongoing compactions.
|
||||
std::set<uint64_t> pending_outputs_ GUARDED_BY(mutex_);
|
||||
|
||||
// Has a background compaction been scheduled or is running?
|
||||
bool background_compaction_scheduled_ GUARDED_BY(mutex_);
|
||||
|
||||
ManualCompaction* manual_compaction_ GUARDED_BY(mutex_);
|
||||
|
||||
VersionSet* const versions_;
|
||||
|
||||
// Have we encountered a background error in paranoid mode?
|
||||
Status bg_error_ GUARDED_BY(mutex_);
|
||||
|
||||
CompactionStats stats_[config::kNumLevels] GUARDED_BY(mutex_);
|
||||
};
|
||||
|
||||
// Sanitize db options. The caller should delete result.info_log if
|
||||
// it is not equal to src.info_log.
|
||||
Options SanitizeOptions(const std::string& db,
|
||||
const InternalKeyComparator* icmp,
|
||||
const InternalFilterPolicy* ipolicy,
|
||||
const Options& src);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_DB_IMPL_H_
|
||||
Generated
+309
@@ -0,0 +1,309 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/db_iter.h"
|
||||
|
||||
#include "db/db_impl.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/filename.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "port/port.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include "util/random.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
#if 0
|
||||
static void DumpInternalIter(Iterator* iter) {
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
ParsedInternalKey k;
|
||||
if (!ParseInternalKey(iter->key(), &k)) {
|
||||
fprintf(stderr, "Corrupt '%s'\n", EscapeString(iter->key()).c_str());
|
||||
} else {
|
||||
fprintf(stderr, "@ '%s'\n", k.DebugString().c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
// Memtables and sstables that make the DB representation contain
|
||||
// (userkey,seq,type) => uservalue entries. DBIter
|
||||
// combines multiple entries for the same userkey found in the DB
|
||||
// representation into a single entry while accounting for sequence
|
||||
// numbers, deletion markers, overwrites, etc.
|
||||
class DBIter : public Iterator {
|
||||
public:
|
||||
// Which direction is the iterator currently moving?
|
||||
// (1) When moving forward, the internal iterator is positioned at
|
||||
// the exact entry that yields this->key(), this->value()
|
||||
// (2) When moving backwards, the internal iterator is positioned
|
||||
// just before all entries whose user key == this->key().
|
||||
enum Direction { kForward, kReverse };
|
||||
|
||||
DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s,
|
||||
uint32_t seed)
|
||||
: db_(db),
|
||||
user_comparator_(cmp),
|
||||
iter_(iter),
|
||||
sequence_(s),
|
||||
direction_(kForward),
|
||||
valid_(false),
|
||||
rnd_(seed),
|
||||
bytes_until_read_sampling_(RandomCompactionPeriod()) {}
|
||||
|
||||
DBIter(const DBIter&) = delete;
|
||||
DBIter& operator=(const DBIter&) = delete;
|
||||
|
||||
virtual ~DBIter() { delete iter_; }
|
||||
virtual bool Valid() const { return valid_; }
|
||||
virtual Slice key() const {
|
||||
assert(valid_);
|
||||
return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_;
|
||||
}
|
||||
virtual Slice value() const {
|
||||
assert(valid_);
|
||||
return (direction_ == kForward) ? iter_->value() : saved_value_;
|
||||
}
|
||||
virtual Status status() const {
|
||||
if (status_.ok()) {
|
||||
return iter_->status();
|
||||
} else {
|
||||
return status_;
|
||||
}
|
||||
}
|
||||
|
||||
virtual void Next();
|
||||
virtual void Prev();
|
||||
virtual void Seek(const Slice& target);
|
||||
virtual void SeekToFirst();
|
||||
virtual void SeekToLast();
|
||||
|
||||
private:
|
||||
void FindNextUserEntry(bool skipping, std::string* skip);
|
||||
void FindPrevUserEntry();
|
||||
bool ParseKey(ParsedInternalKey* key);
|
||||
|
||||
inline void SaveKey(const Slice& k, std::string* dst) {
|
||||
dst->assign(k.data(), k.size());
|
||||
}
|
||||
|
||||
inline void ClearSavedValue() {
|
||||
if (saved_value_.capacity() > 1048576) {
|
||||
std::string empty;
|
||||
swap(empty, saved_value_);
|
||||
} else {
|
||||
saved_value_.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Picks the number of bytes that can be read until a compaction is scheduled.
|
||||
size_t RandomCompactionPeriod() {
|
||||
return rnd_.Uniform(2 * config::kReadBytesPeriod);
|
||||
}
|
||||
|
||||
DBImpl* db_;
|
||||
const Comparator* const user_comparator_;
|
||||
Iterator* const iter_;
|
||||
SequenceNumber const sequence_;
|
||||
Status status_;
|
||||
std::string saved_key_; // == current key when direction_==kReverse
|
||||
std::string saved_value_; // == current raw value when direction_==kReverse
|
||||
Direction direction_;
|
||||
bool valid_;
|
||||
Random rnd_;
|
||||
size_t bytes_until_read_sampling_;
|
||||
};
|
||||
|
||||
inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
|
||||
Slice k = iter_->key();
|
||||
|
||||
size_t bytes_read = k.size() + iter_->value().size();
|
||||
while (bytes_until_read_sampling_ < bytes_read) {
|
||||
bytes_until_read_sampling_ += RandomCompactionPeriod();
|
||||
db_->RecordReadSample(k);
|
||||
}
|
||||
assert(bytes_until_read_sampling_ >= bytes_read);
|
||||
bytes_until_read_sampling_ -= bytes_read;
|
||||
|
||||
if (!ParseInternalKey(k, ikey)) {
|
||||
status_ = Status::Corruption("corrupted internal key in DBIter");
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
void DBIter::Next() {
|
||||
assert(valid_);
|
||||
|
||||
if (direction_ == kReverse) { // Switch directions?
|
||||
direction_ = kForward;
|
||||
// iter_ is pointing just before the entries for this->key(),
|
||||
// so advance into the range of entries for this->key() and then
|
||||
// use the normal skipping code below.
|
||||
if (!iter_->Valid()) {
|
||||
iter_->SeekToFirst();
|
||||
} else {
|
||||
iter_->Next();
|
||||
}
|
||||
if (!iter_->Valid()) {
|
||||
valid_ = false;
|
||||
saved_key_.clear();
|
||||
return;
|
||||
}
|
||||
// saved_key_ already contains the key to skip past.
|
||||
} else {
|
||||
// Store in saved_key_ the current key so we skip it below.
|
||||
SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
|
||||
}
|
||||
|
||||
FindNextUserEntry(true, &saved_key_);
|
||||
}
|
||||
|
||||
void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
|
||||
// Loop until we hit an acceptable entry to yield
|
||||
assert(iter_->Valid());
|
||||
assert(direction_ == kForward);
|
||||
do {
|
||||
ParsedInternalKey ikey;
|
||||
if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
|
||||
switch (ikey.type) {
|
||||
case kTypeDeletion:
|
||||
// Arrange to skip all upcoming entries for this key since
|
||||
// they are hidden by this deletion.
|
||||
SaveKey(ikey.user_key, skip);
|
||||
skipping = true;
|
||||
break;
|
||||
case kTypeValue:
|
||||
if (skipping &&
|
||||
user_comparator_->Compare(ikey.user_key, *skip) <= 0) {
|
||||
// Entry hidden
|
||||
} else {
|
||||
valid_ = true;
|
||||
saved_key_.clear();
|
||||
return;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
iter_->Next();
|
||||
} while (iter_->Valid());
|
||||
saved_key_.clear();
|
||||
valid_ = false;
|
||||
}
|
||||
|
||||
void DBIter::Prev() {
|
||||
assert(valid_);
|
||||
|
||||
if (direction_ == kForward) { // Switch directions?
|
||||
// iter_ is pointing at the current entry. Scan backwards until
|
||||
// the key changes so we can use the normal reverse scanning code.
|
||||
assert(iter_->Valid()); // Otherwise valid_ would have been false
|
||||
SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
|
||||
while (true) {
|
||||
iter_->Prev();
|
||||
if (!iter_->Valid()) {
|
||||
valid_ = false;
|
||||
saved_key_.clear();
|
||||
ClearSavedValue();
|
||||
return;
|
||||
}
|
||||
if (user_comparator_->Compare(ExtractUserKey(iter_->key()), saved_key_) <
|
||||
0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
direction_ = kReverse;
|
||||
}
|
||||
|
||||
FindPrevUserEntry();
|
||||
}
|
||||
|
||||
void DBIter::FindPrevUserEntry() {
|
||||
assert(direction_ == kReverse);
|
||||
|
||||
ValueType value_type = kTypeDeletion;
|
||||
if (iter_->Valid()) {
|
||||
do {
|
||||
ParsedInternalKey ikey;
|
||||
if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
|
||||
if ((value_type != kTypeDeletion) &&
|
||||
user_comparator_->Compare(ikey.user_key, saved_key_) < 0) {
|
||||
// We encountered a non-deleted value in entries for previous keys,
|
||||
break;
|
||||
}
|
||||
value_type = ikey.type;
|
||||
if (value_type == kTypeDeletion) {
|
||||
saved_key_.clear();
|
||||
ClearSavedValue();
|
||||
} else {
|
||||
Slice raw_value = iter_->value();
|
||||
if (saved_value_.capacity() > raw_value.size() + 1048576) {
|
||||
std::string empty;
|
||||
swap(empty, saved_value_);
|
||||
}
|
||||
SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
|
||||
saved_value_.assign(raw_value.data(), raw_value.size());
|
||||
}
|
||||
}
|
||||
iter_->Prev();
|
||||
} while (iter_->Valid());
|
||||
}
|
||||
|
||||
if (value_type == kTypeDeletion) {
|
||||
// End
|
||||
valid_ = false;
|
||||
saved_key_.clear();
|
||||
ClearSavedValue();
|
||||
direction_ = kForward;
|
||||
} else {
|
||||
valid_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
void DBIter::Seek(const Slice& target) {
|
||||
direction_ = kForward;
|
||||
ClearSavedValue();
|
||||
saved_key_.clear();
|
||||
AppendInternalKey(&saved_key_,
|
||||
ParsedInternalKey(target, sequence_, kValueTypeForSeek));
|
||||
iter_->Seek(saved_key_);
|
||||
if (iter_->Valid()) {
|
||||
FindNextUserEntry(false, &saved_key_ /* temporary storage */);
|
||||
} else {
|
||||
valid_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
void DBIter::SeekToFirst() {
|
||||
direction_ = kForward;
|
||||
ClearSavedValue();
|
||||
iter_->SeekToFirst();
|
||||
if (iter_->Valid()) {
|
||||
FindNextUserEntry(false, &saved_key_ /* temporary storage */);
|
||||
} else {
|
||||
valid_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
void DBIter::SeekToLast() {
|
||||
direction_ = kReverse;
|
||||
ClearSavedValue();
|
||||
iter_->SeekToLast();
|
||||
FindPrevUserEntry();
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator,
|
||||
Iterator* internal_iter, SequenceNumber sequence,
|
||||
uint32_t seed) {
|
||||
return new DBIter(db, user_key_comparator, internal_iter, sequence, seed);
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
Generated
+26
@@ -0,0 +1,26 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_DB_ITER_H_
|
||||
#define STORAGE_LEVELDB_DB_DB_ITER_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/db.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class DBImpl;
|
||||
|
||||
// Return a new iterator that converts internal keys (yielded by
|
||||
// "*internal_iter") that were live at the specified "sequence" number
|
||||
// into appropriate user keys.
|
||||
Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator,
|
||||
Iterator* internal_iter, SequenceNumber sequence,
|
||||
uint32_t seed);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_DB_ITER_H_
|
||||
Generated
+140
@@ -0,0 +1,140 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/dbformat.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "port/port.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
static uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {
|
||||
assert(seq <= kMaxSequenceNumber);
|
||||
assert(t <= kValueTypeForSeek);
|
||||
return (seq << 8) | t;
|
||||
}
|
||||
|
||||
void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
|
||||
result->append(key.user_key.data(), key.user_key.size());
|
||||
PutFixed64(result, PackSequenceAndType(key.sequence, key.type));
|
||||
}
|
||||
|
||||
std::string ParsedInternalKey::DebugString() const {
|
||||
char buf[50];
|
||||
snprintf(buf, sizeof(buf), "' @ %llu : %d", (unsigned long long)sequence,
|
||||
int(type));
|
||||
std::string result = "'";
|
||||
result += EscapeString(user_key.ToString());
|
||||
result += buf;
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string InternalKey::DebugString() const {
|
||||
std::string result;
|
||||
ParsedInternalKey parsed;
|
||||
if (ParseInternalKey(rep_, &parsed)) {
|
||||
result = parsed.DebugString();
|
||||
} else {
|
||||
result = "(bad)";
|
||||
result.append(EscapeString(rep_));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const char* InternalKeyComparator::Name() const {
|
||||
return "leveldb.InternalKeyComparator";
|
||||
}
|
||||
|
||||
int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
|
||||
// Order by:
|
||||
// increasing user key (according to user-supplied comparator)
|
||||
// decreasing sequence number
|
||||
// decreasing type (though sequence# should be enough to disambiguate)
|
||||
int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
|
||||
if (r == 0) {
|
||||
const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
|
||||
const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
|
||||
if (anum > bnum) {
|
||||
r = -1;
|
||||
} else if (anum < bnum) {
|
||||
r = +1;
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
void InternalKeyComparator::FindShortestSeparator(std::string* start,
|
||||
const Slice& limit) const {
|
||||
// Attempt to shorten the user portion of the key
|
||||
Slice user_start = ExtractUserKey(*start);
|
||||
Slice user_limit = ExtractUserKey(limit);
|
||||
std::string tmp(user_start.data(), user_start.size());
|
||||
user_comparator_->FindShortestSeparator(&tmp, user_limit);
|
||||
if (tmp.size() < user_start.size() &&
|
||||
user_comparator_->Compare(user_start, tmp) < 0) {
|
||||
// User key has become shorter physically, but larger logically.
|
||||
// Tack on the earliest possible number to the shortened user key.
|
||||
PutFixed64(&tmp,
|
||||
PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek));
|
||||
assert(this->Compare(*start, tmp) < 0);
|
||||
assert(this->Compare(tmp, limit) < 0);
|
||||
start->swap(tmp);
|
||||
}
|
||||
}
|
||||
|
||||
void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
|
||||
Slice user_key = ExtractUserKey(*key);
|
||||
std::string tmp(user_key.data(), user_key.size());
|
||||
user_comparator_->FindShortSuccessor(&tmp);
|
||||
if (tmp.size() < user_key.size() &&
|
||||
user_comparator_->Compare(user_key, tmp) < 0) {
|
||||
// User key has become shorter physically, but larger logically.
|
||||
// Tack on the earliest possible number to the shortened user key.
|
||||
PutFixed64(&tmp,
|
||||
PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek));
|
||||
assert(this->Compare(*key, tmp) < 0);
|
||||
key->swap(tmp);
|
||||
}
|
||||
}
|
||||
|
||||
const char* InternalFilterPolicy::Name() const { return user_policy_->Name(); }
|
||||
|
||||
void InternalFilterPolicy::CreateFilter(const Slice* keys, int n,
|
||||
std::string* dst) const {
|
||||
// We rely on the fact that the code in table.cc does not mind us
|
||||
// adjusting keys[].
|
||||
Slice* mkey = const_cast<Slice*>(keys);
|
||||
for (int i = 0; i < n; i++) {
|
||||
mkey[i] = ExtractUserKey(keys[i]);
|
||||
// TODO(sanjay): Suppress dups?
|
||||
}
|
||||
user_policy_->CreateFilter(keys, n, dst);
|
||||
}
|
||||
|
||||
bool InternalFilterPolicy::KeyMayMatch(const Slice& key, const Slice& f) const {
|
||||
return user_policy_->KeyMayMatch(ExtractUserKey(key), f);
|
||||
}
|
||||
|
||||
LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
|
||||
size_t usize = user_key.size();
|
||||
size_t needed = usize + 13; // A conservative estimate
|
||||
char* dst;
|
||||
if (needed <= sizeof(space_)) {
|
||||
dst = space_;
|
||||
} else {
|
||||
dst = new char[needed];
|
||||
}
|
||||
start_ = dst;
|
||||
dst = EncodeVarint32(dst, usize + 8);
|
||||
kstart_ = dst;
|
||||
memcpy(dst, user_key.data(), usize);
|
||||
dst += usize;
|
||||
EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek));
|
||||
dst += 8;
|
||||
end_ = dst;
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
Generated
+218
@@ -0,0 +1,218 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
#define STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/filter_policy.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/table_builder.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/logging.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
// Grouping of constants. We may want to make some of these
|
||||
// parameters set via options.
|
||||
namespace config {
|
||||
static const int kNumLevels = 7;
|
||||
|
||||
// Level-0 compaction is started when we hit this many files.
|
||||
static const int kL0_CompactionTrigger = 4;
|
||||
|
||||
// Soft limit on number of level-0 files. We slow down writes at this point.
|
||||
static const int kL0_SlowdownWritesTrigger = 8;
|
||||
|
||||
// Maximum number of level-0 files. We stop writes at this point.
|
||||
static const int kL0_StopWritesTrigger = 12;
|
||||
|
||||
// Maximum level to which a new compacted memtable is pushed if it
|
||||
// does not create overlap. We try to push to level 2 to avoid the
|
||||
// relatively expensive level 0=>1 compactions and to avoid some
|
||||
// expensive manifest file operations. We do not push all the way to
|
||||
// the largest level since that can generate a lot of wasted disk
|
||||
// space if the same key space is being repeatedly overwritten.
|
||||
static const int kMaxMemCompactLevel = 2;
|
||||
|
||||
// Approximate gap in bytes between samples of data read during iteration.
|
||||
static const int kReadBytesPeriod = 1048576;
|
||||
|
||||
} // namespace config
|
||||
|
||||
class InternalKey;
|
||||
|
||||
// Value types encoded as the last component of internal keys.
|
||||
// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk
|
||||
// data structures.
|
||||
enum ValueType { kTypeDeletion = 0x0, kTypeValue = 0x1 };
|
||||
// kValueTypeForSeek defines the ValueType that should be passed when
|
||||
// constructing a ParsedInternalKey object for seeking to a particular
|
||||
// sequence number (since we sort sequence numbers in decreasing order
|
||||
// and the value type is embedded as the low 8 bits in the sequence
|
||||
// number in internal keys, we need to use the highest-numbered
|
||||
// ValueType, not the lowest).
|
||||
static const ValueType kValueTypeForSeek = kTypeValue;
|
||||
|
||||
typedef uint64_t SequenceNumber;
|
||||
|
||||
// We leave eight bits empty at the bottom so a type and sequence#
|
||||
// can be packed together into 64-bits.
|
||||
static const SequenceNumber kMaxSequenceNumber = ((0x1ull << 56) - 1);
|
||||
|
||||
struct ParsedInternalKey {
|
||||
Slice user_key;
|
||||
SequenceNumber sequence;
|
||||
ValueType type;
|
||||
|
||||
ParsedInternalKey() {} // Intentionally left uninitialized (for speed)
|
||||
ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
|
||||
: user_key(u), sequence(seq), type(t) {}
|
||||
std::string DebugString() const;
|
||||
};
|
||||
|
||||
// Return the length of the encoding of "key".
|
||||
inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) {
|
||||
return key.user_key.size() + 8;
|
||||
}
|
||||
|
||||
// Append the serialization of "key" to *result.
|
||||
void AppendInternalKey(std::string* result, const ParsedInternalKey& key);
|
||||
|
||||
// Attempt to parse an internal key from "internal_key". On success,
|
||||
// stores the parsed data in "*result", and returns true.
|
||||
//
|
||||
// On error, returns false, leaves "*result" in an undefined state.
|
||||
bool ParseInternalKey(const Slice& internal_key, ParsedInternalKey* result);
|
||||
|
||||
// Returns the user key portion of an internal key.
|
||||
inline Slice ExtractUserKey(const Slice& internal_key) {
|
||||
assert(internal_key.size() >= 8);
|
||||
return Slice(internal_key.data(), internal_key.size() - 8);
|
||||
}
|
||||
|
||||
// A comparator for internal keys that uses a specified comparator for
|
||||
// the user key portion and breaks ties by decreasing sequence number.
|
||||
class InternalKeyComparator : public Comparator {
|
||||
private:
|
||||
const Comparator* user_comparator_;
|
||||
|
||||
public:
|
||||
explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) {}
|
||||
virtual const char* Name() const;
|
||||
virtual int Compare(const Slice& a, const Slice& b) const;
|
||||
virtual void FindShortestSeparator(std::string* start,
|
||||
const Slice& limit) const;
|
||||
virtual void FindShortSuccessor(std::string* key) const;
|
||||
|
||||
const Comparator* user_comparator() const { return user_comparator_; }
|
||||
|
||||
int Compare(const InternalKey& a, const InternalKey& b) const;
|
||||
};
|
||||
|
||||
// Filter policy wrapper that converts from internal keys to user keys
|
||||
class InternalFilterPolicy : public FilterPolicy {
|
||||
private:
|
||||
const FilterPolicy* const user_policy_;
|
||||
|
||||
public:
|
||||
explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) {}
|
||||
virtual const char* Name() const;
|
||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const;
|
||||
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const;
|
||||
};
|
||||
|
||||
// Modules in this directory should keep internal keys wrapped inside
|
||||
// the following class instead of plain strings so that we do not
|
||||
// incorrectly use string comparisons instead of an InternalKeyComparator.
|
||||
class InternalKey {
|
||||
private:
|
||||
std::string rep_;
|
||||
|
||||
public:
|
||||
InternalKey() {} // Leave rep_ as empty to indicate it is invalid
|
||||
InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) {
|
||||
AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t));
|
||||
}
|
||||
|
||||
void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); }
|
||||
Slice Encode() const {
|
||||
assert(!rep_.empty());
|
||||
return rep_;
|
||||
}
|
||||
|
||||
Slice user_key() const { return ExtractUserKey(rep_); }
|
||||
|
||||
void SetFrom(const ParsedInternalKey& p) {
|
||||
rep_.clear();
|
||||
AppendInternalKey(&rep_, p);
|
||||
}
|
||||
|
||||
void Clear() { rep_.clear(); }
|
||||
|
||||
std::string DebugString() const;
|
||||
};
|
||||
|
||||
inline int InternalKeyComparator::Compare(const InternalKey& a,
|
||||
const InternalKey& b) const {
|
||||
return Compare(a.Encode(), b.Encode());
|
||||
}
|
||||
|
||||
inline bool ParseInternalKey(const Slice& internal_key,
|
||||
ParsedInternalKey* result) {
|
||||
const size_t n = internal_key.size();
|
||||
if (n < 8) return false;
|
||||
uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
|
||||
unsigned char c = num & 0xff;
|
||||
result->sequence = num >> 8;
|
||||
result->type = static_cast<ValueType>(c);
|
||||
result->user_key = Slice(internal_key.data(), n - 8);
|
||||
return (c <= static_cast<unsigned char>(kTypeValue));
|
||||
}
|
||||
|
||||
// A helper class useful for DBImpl::Get()
|
||||
class LookupKey {
|
||||
public:
|
||||
// Initialize *this for looking up user_key at a snapshot with
|
||||
// the specified sequence number.
|
||||
LookupKey(const Slice& user_key, SequenceNumber sequence);
|
||||
|
||||
LookupKey(const LookupKey&) = delete;
|
||||
LookupKey& operator=(const LookupKey&) = delete;
|
||||
|
||||
~LookupKey();
|
||||
|
||||
// Return a key suitable for lookup in a MemTable.
|
||||
Slice memtable_key() const { return Slice(start_, end_ - start_); }
|
||||
|
||||
// Return an internal key (suitable for passing to an internal iterator)
|
||||
Slice internal_key() const { return Slice(kstart_, end_ - kstart_); }
|
||||
|
||||
// Return the user key
|
||||
Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); }
|
||||
|
||||
private:
|
||||
// We construct a char array of the form:
|
||||
// klength varint32 <-- start_
|
||||
// userkey char[klength] <-- kstart_
|
||||
// tag uint64
|
||||
// <-- end_
|
||||
// The array is a suitable MemTable key.
|
||||
// The suffix starting with "userkey" can be used as an InternalKey.
|
||||
const char* start_;
|
||||
const char* kstart_;
|
||||
const char* end_;
|
||||
char space_[200]; // Avoid allocation for short keys
|
||||
};
|
||||
|
||||
inline LookupKey::~LookupKey() {
|
||||
if (start_ != space_) delete[] start_;
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
Generated
+232
@@ -0,0 +1,232 @@
|
||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "leveldb/dumpfile.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/log_reader.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "util/logging.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
namespace {
|
||||
|
||||
bool GuessType(const std::string& fname, FileType* type) {
|
||||
size_t pos = fname.rfind('/');
|
||||
std::string basename;
|
||||
if (pos == std::string::npos) {
|
||||
basename = fname;
|
||||
} else {
|
||||
basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1);
|
||||
}
|
||||
uint64_t ignored;
|
||||
return ParseFileName(basename, &ignored, type);
|
||||
}
|
||||
|
||||
// Notified when log reader encounters corruption.
|
||||
class CorruptionReporter : public log::Reader::Reporter {
|
||||
public:
|
||||
virtual void Corruption(size_t bytes, const Status& status) {
|
||||
std::string r = "corruption: ";
|
||||
AppendNumberTo(&r, bytes);
|
||||
r += " bytes; ";
|
||||
r += status.ToString();
|
||||
r.push_back('\n');
|
||||
dst_->Append(r);
|
||||
}
|
||||
|
||||
WritableFile* dst_;
|
||||
};
|
||||
|
||||
// Print contents of a log file. (*func)() is called on every record.
|
||||
Status PrintLogContents(Env* env, const std::string& fname,
|
||||
void (*func)(uint64_t, Slice, WritableFile*),
|
||||
WritableFile* dst) {
|
||||
SequentialFile* file;
|
||||
Status s = env->NewSequentialFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
CorruptionReporter reporter;
|
||||
reporter.dst_ = dst;
|
||||
log::Reader reader(file, &reporter, true, 0);
|
||||
Slice record;
|
||||
std::string scratch;
|
||||
while (reader.ReadRecord(&record, &scratch)) {
|
||||
(*func)(reader.LastRecordOffset(), record, dst);
|
||||
}
|
||||
delete file;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// Called on every item found in a WriteBatch.
|
||||
class WriteBatchItemPrinter : public WriteBatch::Handler {
|
||||
public:
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
std::string r = " put '";
|
||||
AppendEscapedStringTo(&r, key);
|
||||
r += "' '";
|
||||
AppendEscapedStringTo(&r, value);
|
||||
r += "'\n";
|
||||
dst_->Append(r);
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
std::string r = " del '";
|
||||
AppendEscapedStringTo(&r, key);
|
||||
r += "'\n";
|
||||
dst_->Append(r);
|
||||
}
|
||||
|
||||
WritableFile* dst_;
|
||||
};
|
||||
|
||||
// Called on every log record (each one of which is a WriteBatch)
|
||||
// found in a kLogFile.
|
||||
static void WriteBatchPrinter(uint64_t pos, Slice record, WritableFile* dst) {
|
||||
std::string r = "--- offset ";
|
||||
AppendNumberTo(&r, pos);
|
||||
r += "; ";
|
||||
if (record.size() < 12) {
|
||||
r += "log record length ";
|
||||
AppendNumberTo(&r, record.size());
|
||||
r += " is too small\n";
|
||||
dst->Append(r);
|
||||
return;
|
||||
}
|
||||
WriteBatch batch;
|
||||
WriteBatchInternal::SetContents(&batch, record);
|
||||
r += "sequence ";
|
||||
AppendNumberTo(&r, WriteBatchInternal::Sequence(&batch));
|
||||
r.push_back('\n');
|
||||
dst->Append(r);
|
||||
WriteBatchItemPrinter batch_item_printer;
|
||||
batch_item_printer.dst_ = dst;
|
||||
Status s = batch.Iterate(&batch_item_printer);
|
||||
if (!s.ok()) {
|
||||
dst->Append(" error: " + s.ToString() + "\n");
|
||||
}
|
||||
}
|
||||
|
||||
Status DumpLog(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
return PrintLogContents(env, fname, WriteBatchPrinter, dst);
|
||||
}
|
||||
|
||||
// Called on every log record (each one of which is a WriteBatch)
|
||||
// found in a kDescriptorFile.
|
||||
static void VersionEditPrinter(uint64_t pos, Slice record, WritableFile* dst) {
|
||||
std::string r = "--- offset ";
|
||||
AppendNumberTo(&r, pos);
|
||||
r += "; ";
|
||||
VersionEdit edit;
|
||||
Status s = edit.DecodeFrom(record);
|
||||
if (!s.ok()) {
|
||||
r += s.ToString();
|
||||
r.push_back('\n');
|
||||
} else {
|
||||
r += edit.DebugString();
|
||||
}
|
||||
dst->Append(r);
|
||||
}
|
||||
|
||||
Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
return PrintLogContents(env, fname, VersionEditPrinter, dst);
|
||||
}
|
||||
|
||||
Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
uint64_t file_size;
|
||||
RandomAccessFile* file = nullptr;
|
||||
Table* table = nullptr;
|
||||
Status s = env->GetFileSize(fname, &file_size);
|
||||
if (s.ok()) {
|
||||
s = env->NewRandomAccessFile(fname, &file);
|
||||
}
|
||||
if (s.ok()) {
|
||||
// We use the default comparator, which may or may not match the
|
||||
// comparator used in this database. However this should not cause
|
||||
// problems since we only use Table operations that do not require
|
||||
// any comparisons. In particular, we do not call Seek or Prev.
|
||||
s = Table::Open(Options(), file, file_size, &table);
|
||||
}
|
||||
if (!s.ok()) {
|
||||
delete table;
|
||||
delete file;
|
||||
return s;
|
||||
}
|
||||
|
||||
ReadOptions ro;
|
||||
ro.fill_cache = false;
|
||||
Iterator* iter = table->NewIterator(ro);
|
||||
std::string r;
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
r.clear();
|
||||
ParsedInternalKey key;
|
||||
if (!ParseInternalKey(iter->key(), &key)) {
|
||||
r = "badkey '";
|
||||
AppendEscapedStringTo(&r, iter->key());
|
||||
r += "' => '";
|
||||
AppendEscapedStringTo(&r, iter->value());
|
||||
r += "'\n";
|
||||
dst->Append(r);
|
||||
} else {
|
||||
r = "'";
|
||||
AppendEscapedStringTo(&r, key.user_key);
|
||||
r += "' @ ";
|
||||
AppendNumberTo(&r, key.sequence);
|
||||
r += " : ";
|
||||
if (key.type == kTypeDeletion) {
|
||||
r += "del";
|
||||
} else if (key.type == kTypeValue) {
|
||||
r += "val";
|
||||
} else {
|
||||
AppendNumberTo(&r, key.type);
|
||||
}
|
||||
r += " => '";
|
||||
AppendEscapedStringTo(&r, iter->value());
|
||||
r += "'\n";
|
||||
dst->Append(r);
|
||||
}
|
||||
}
|
||||
s = iter->status();
|
||||
if (!s.ok()) {
|
||||
dst->Append("iterator error: " + s.ToString() + "\n");
|
||||
}
|
||||
|
||||
delete iter;
|
||||
delete table;
|
||||
delete file;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
FileType ftype;
|
||||
if (!GuessType(fname, &ftype)) {
|
||||
return Status::InvalidArgument(fname + ": unknown file type");
|
||||
}
|
||||
switch (ftype) {
|
||||
case kLogFile:
|
||||
return DumpLog(env, fname, dst);
|
||||
case kDescriptorFile:
|
||||
return DumpDescriptor(env, fname, dst);
|
||||
case kTableFile:
|
||||
return DumpTable(env, fname, dst);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return Status::InvalidArgument(fname + ": not a dump-able file type");
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
Generated
+141
@@ -0,0 +1,141 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/filename.h"
|
||||
|
||||
#include <ctype.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/logging.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
// A utility routine: write "data" to the named file and Sync() it.
|
||||
Status WriteStringToFileSync(Env* env, const Slice& data,
|
||||
const std::string& fname);
|
||||
|
||||
static std::string MakeFileName(const std::string& dbname, uint64_t number,
|
||||
const char* suffix) {
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "/%06llu.%s",
|
||||
static_cast<unsigned long long>(number), suffix);
|
||||
return dbname + buf;
|
||||
}
|
||||
|
||||
std::string LogFileName(const std::string& dbname, uint64_t number) {
|
||||
assert(number > 0);
|
||||
return MakeFileName(dbname, number, "log");
|
||||
}
|
||||
|
||||
std::string TableFileName(const std::string& dbname, uint64_t number) {
|
||||
assert(number > 0);
|
||||
return MakeFileName(dbname, number, "ldb");
|
||||
}
|
||||
|
||||
std::string SSTTableFileName(const std::string& dbname, uint64_t number) {
|
||||
assert(number > 0);
|
||||
return MakeFileName(dbname, number, "sst");
|
||||
}
|
||||
|
||||
std::string DescriptorFileName(const std::string& dbname, uint64_t number) {
|
||||
assert(number > 0);
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "/MANIFEST-%06llu",
|
||||
static_cast<unsigned long long>(number));
|
||||
return dbname + buf;
|
||||
}
|
||||
|
||||
std::string CurrentFileName(const std::string& dbname) {
|
||||
return dbname + "/CURRENT";
|
||||
}
|
||||
|
||||
std::string LockFileName(const std::string& dbname) { return dbname + "/LOCK"; }
|
||||
|
||||
std::string TempFileName(const std::string& dbname, uint64_t number) {
|
||||
assert(number > 0);
|
||||
return MakeFileName(dbname, number, "dbtmp");
|
||||
}
|
||||
|
||||
std::string InfoLogFileName(const std::string& dbname) {
|
||||
return dbname + "/LOG";
|
||||
}
|
||||
|
||||
// Return the name of the old info log file for "dbname".
|
||||
std::string OldInfoLogFileName(const std::string& dbname) {
|
||||
return dbname + "/LOG.old";
|
||||
}
|
||||
|
||||
// Owned filenames have the form:
|
||||
// dbname/CURRENT
|
||||
// dbname/LOCK
|
||||
// dbname/LOG
|
||||
// dbname/LOG.old
|
||||
// dbname/MANIFEST-[0-9]+
|
||||
// dbname/[0-9]+.(log|sst|ldb)
|
||||
bool ParseFileName(const std::string& filename, uint64_t* number,
|
||||
FileType* type) {
|
||||
Slice rest(filename);
|
||||
if (rest == "CURRENT") {
|
||||
*number = 0;
|
||||
*type = kCurrentFile;
|
||||
} else if (rest == "LOCK") {
|
||||
*number = 0;
|
||||
*type = kDBLockFile;
|
||||
} else if (rest == "LOG" || rest == "LOG.old") {
|
||||
*number = 0;
|
||||
*type = kInfoLogFile;
|
||||
} else if (rest.starts_with("MANIFEST-")) {
|
||||
rest.remove_prefix(strlen("MANIFEST-"));
|
||||
uint64_t num;
|
||||
if (!ConsumeDecimalNumber(&rest, &num)) {
|
||||
return false;
|
||||
}
|
||||
if (!rest.empty()) {
|
||||
return false;
|
||||
}
|
||||
*type = kDescriptorFile;
|
||||
*number = num;
|
||||
} else {
|
||||
// Avoid strtoull() to keep filename format independent of the
|
||||
// current locale
|
||||
uint64_t num;
|
||||
if (!ConsumeDecimalNumber(&rest, &num)) {
|
||||
return false;
|
||||
}
|
||||
Slice suffix = rest;
|
||||
if (suffix == Slice(".log")) {
|
||||
*type = kLogFile;
|
||||
} else if (suffix == Slice(".sst") || suffix == Slice(".ldb")) {
|
||||
*type = kTableFile;
|
||||
} else if (suffix == Slice(".dbtmp")) {
|
||||
*type = kTempFile;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
*number = num;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
Status SetCurrentFile(Env* env, const std::string& dbname,
|
||||
uint64_t descriptor_number) {
|
||||
// Remove leading "dbname/" and add newline to manifest file name
|
||||
std::string manifest = DescriptorFileName(dbname, descriptor_number);
|
||||
Slice contents = manifest;
|
||||
assert(contents.starts_with(dbname + "/"));
|
||||
contents.remove_prefix(dbname.size() + 1);
|
||||
std::string tmp = TempFileName(dbname, descriptor_number);
|
||||
Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp);
|
||||
if (s.ok()) {
|
||||
s = env->RenameFile(tmp, CurrentFileName(dbname));
|
||||
}
|
||||
if (!s.ok()) {
|
||||
env->DeleteFile(tmp);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
Generated
+84
@@ -0,0 +1,84 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// File names used by DB code
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_FILENAME_H_
|
||||
#define STORAGE_LEVELDB_DB_FILENAME_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "port/port.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class Env;
|
||||
|
||||
enum FileType {
|
||||
kLogFile,
|
||||
kDBLockFile,
|
||||
kTableFile,
|
||||
kDescriptorFile,
|
||||
kCurrentFile,
|
||||
kTempFile,
|
||||
kInfoLogFile // Either the current one, or an old one
|
||||
};
|
||||
|
||||
// Return the name of the log file with the specified number
|
||||
// in the db named by "dbname". The result will be prefixed with
|
||||
// "dbname".
|
||||
std::string LogFileName(const std::string& dbname, uint64_t number);
|
||||
|
||||
// Return the name of the sstable with the specified number
|
||||
// in the db named by "dbname". The result will be prefixed with
|
||||
// "dbname".
|
||||
std::string TableFileName(const std::string& dbname, uint64_t number);
|
||||
|
||||
// Return the legacy file name for an sstable with the specified number
|
||||
// in the db named by "dbname". The result will be prefixed with
|
||||
// "dbname".
|
||||
std::string SSTTableFileName(const std::string& dbname, uint64_t number);
|
||||
|
||||
// Return the name of the descriptor file for the db named by
|
||||
// "dbname" and the specified incarnation number. The result will be
|
||||
// prefixed with "dbname".
|
||||
std::string DescriptorFileName(const std::string& dbname, uint64_t number);
|
||||
|
||||
// Return the name of the current file. This file contains the name
|
||||
// of the current manifest file. The result will be prefixed with
|
||||
// "dbname".
|
||||
std::string CurrentFileName(const std::string& dbname);
|
||||
|
||||
// Return the name of the lock file for the db named by
|
||||
// "dbname". The result will be prefixed with "dbname".
|
||||
std::string LockFileName(const std::string& dbname);
|
||||
|
||||
// Return the name of a temporary file owned by the db named "dbname".
|
||||
// The result will be prefixed with "dbname".
|
||||
std::string TempFileName(const std::string& dbname, uint64_t number);
|
||||
|
||||
// Return the name of the info log file for "dbname".
|
||||
std::string InfoLogFileName(const std::string& dbname);
|
||||
|
||||
// Return the name of the old info log file for "dbname".
|
||||
std::string OldInfoLogFileName(const std::string& dbname);
|
||||
|
||||
// If filename is a leveldb file, store the type of the file in *type.
|
||||
// The number encoded in the filename is stored in *number. If the
|
||||
// filename was successfully parsed, returns true. Else return false.
|
||||
bool ParseFileName(const std::string& filename, uint64_t* number,
|
||||
FileType* type);
|
||||
|
||||
// Make the CURRENT file point to the descriptor file with the
|
||||
// specified number.
|
||||
Status SetCurrentFile(Env* env, const std::string& dbname,
|
||||
uint64_t descriptor_number);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_FILENAME_H_
|
||||
Generated
+35
@@ -0,0 +1,35 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// Log format information shared by reader and writer.
|
||||
// See ../doc/log_format.md for more detail.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_
|
||||
#define STORAGE_LEVELDB_DB_LOG_FORMAT_H_
|
||||
|
||||
namespace leveldb {
|
||||
namespace log {
|
||||
|
||||
enum RecordType {
|
||||
// Zero is reserved for preallocated files
|
||||
kZeroType = 0,
|
||||
|
||||
kFullType = 1,
|
||||
|
||||
// For fragments
|
||||
kFirstType = 2,
|
||||
kMiddleType = 3,
|
||||
kLastType = 4
|
||||
};
|
||||
static const int kMaxRecordType = kLastType;
|
||||
|
||||
static const int kBlockSize = 32768;
|
||||
|
||||
// Header is checksum (4 bytes), length (2 bytes), type (1 byte).
|
||||
static const int kHeaderSize = 4 + 2 + 1;
|
||||
|
||||
} // namespace log
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_
|
||||
Generated
+274
@@ -0,0 +1,274 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/log_reader.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "leveldb/env.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/crc32c.h"
|
||||
|
||||
namespace leveldb {
|
||||
namespace log {
|
||||
|
||||
Reader::Reporter::~Reporter() {}
|
||||
|
||||
Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
|
||||
uint64_t initial_offset)
|
||||
: file_(file),
|
||||
reporter_(reporter),
|
||||
checksum_(checksum),
|
||||
backing_store_(new char[kBlockSize]),
|
||||
buffer_(),
|
||||
eof_(false),
|
||||
last_record_offset_(0),
|
||||
end_of_buffer_offset_(0),
|
||||
initial_offset_(initial_offset),
|
||||
resyncing_(initial_offset > 0) {}
|
||||
|
||||
Reader::~Reader() { delete[] backing_store_; }
|
||||
|
||||
bool Reader::SkipToInitialBlock() {
|
||||
const size_t offset_in_block = initial_offset_ % kBlockSize;
|
||||
uint64_t block_start_location = initial_offset_ - offset_in_block;
|
||||
|
||||
// Don't search a block if we'd be in the trailer
|
||||
if (offset_in_block > kBlockSize - 6) {
|
||||
block_start_location += kBlockSize;
|
||||
}
|
||||
|
||||
end_of_buffer_offset_ = block_start_location;
|
||||
|
||||
// Skip to start of first block that can contain the initial record
|
||||
if (block_start_location > 0) {
|
||||
Status skip_status = file_->Skip(block_start_location);
|
||||
if (!skip_status.ok()) {
|
||||
ReportDrop(block_start_location, skip_status);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Reader::ReadRecord(Slice* record, std::string* scratch) {
|
||||
if (last_record_offset_ < initial_offset_) {
|
||||
if (!SkipToInitialBlock()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
scratch->clear();
|
||||
record->clear();
|
||||
bool in_fragmented_record = false;
|
||||
// Record offset of the logical record that we're reading
|
||||
// 0 is a dummy value to make compilers happy
|
||||
uint64_t prospective_record_offset = 0;
|
||||
|
||||
Slice fragment;
|
||||
while (true) {
|
||||
const unsigned int record_type = ReadPhysicalRecord(&fragment);
|
||||
|
||||
// ReadPhysicalRecord may have only had an empty trailer remaining in its
|
||||
// internal buffer. Calculate the offset of the next physical record now
|
||||
// that it has returned, properly accounting for its header size.
|
||||
uint64_t physical_record_offset =
|
||||
end_of_buffer_offset_ - buffer_.size() - kHeaderSize - fragment.size();
|
||||
|
||||
if (resyncing_) {
|
||||
if (record_type == kMiddleType) {
|
||||
continue;
|
||||
} else if (record_type == kLastType) {
|
||||
resyncing_ = false;
|
||||
continue;
|
||||
} else {
|
||||
resyncing_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
switch (record_type) {
|
||||
case kFullType:
|
||||
if (in_fragmented_record) {
|
||||
// Handle bug in earlier versions of log::Writer where
|
||||
// it could emit an empty kFirstType record at the tail end
|
||||
// of a block followed by a kFullType or kFirstType record
|
||||
// at the beginning of the next block.
|
||||
if (!scratch->empty()) {
|
||||
ReportCorruption(scratch->size(), "partial record without end(1)");
|
||||
}
|
||||
}
|
||||
prospective_record_offset = physical_record_offset;
|
||||
scratch->clear();
|
||||
*record = fragment;
|
||||
last_record_offset_ = prospective_record_offset;
|
||||
return true;
|
||||
|
||||
case kFirstType:
|
||||
if (in_fragmented_record) {
|
||||
// Handle bug in earlier versions of log::Writer where
|
||||
// it could emit an empty kFirstType record at the tail end
|
||||
// of a block followed by a kFullType or kFirstType record
|
||||
// at the beginning of the next block.
|
||||
if (!scratch->empty()) {
|
||||
ReportCorruption(scratch->size(), "partial record without end(2)");
|
||||
}
|
||||
}
|
||||
prospective_record_offset = physical_record_offset;
|
||||
scratch->assign(fragment.data(), fragment.size());
|
||||
in_fragmented_record = true;
|
||||
break;
|
||||
|
||||
case kMiddleType:
|
||||
if (!in_fragmented_record) {
|
||||
ReportCorruption(fragment.size(),
|
||||
"missing start of fragmented record(1)");
|
||||
} else {
|
||||
scratch->append(fragment.data(), fragment.size());
|
||||
}
|
||||
break;
|
||||
|
||||
case kLastType:
|
||||
if (!in_fragmented_record) {
|
||||
ReportCorruption(fragment.size(),
|
||||
"missing start of fragmented record(2)");
|
||||
} else {
|
||||
scratch->append(fragment.data(), fragment.size());
|
||||
*record = Slice(*scratch);
|
||||
last_record_offset_ = prospective_record_offset;
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
|
||||
case kEof:
|
||||
if (in_fragmented_record) {
|
||||
// This can be caused by the writer dying immediately after
|
||||
// writing a physical record but before completing the next; don't
|
||||
// treat it as a corruption, just ignore the entire logical record.
|
||||
scratch->clear();
|
||||
}
|
||||
return false;
|
||||
|
||||
case kBadRecord:
|
||||
if (in_fragmented_record) {
|
||||
ReportCorruption(scratch->size(), "error in middle of record");
|
||||
in_fragmented_record = false;
|
||||
scratch->clear();
|
||||
}
|
||||
break;
|
||||
|
||||
default: {
|
||||
char buf[40];
|
||||
snprintf(buf, sizeof(buf), "unknown record type %u", record_type);
|
||||
ReportCorruption(
|
||||
(fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
|
||||
buf);
|
||||
in_fragmented_record = false;
|
||||
scratch->clear();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
uint64_t Reader::LastRecordOffset() { return last_record_offset_; }
|
||||
|
||||
void Reader::ReportCorruption(uint64_t bytes, const char* reason) {
|
||||
ReportDrop(bytes, Status::Corruption(reason));
|
||||
}
|
||||
|
||||
void Reader::ReportDrop(uint64_t bytes, const Status& reason) {
|
||||
if (reporter_ != nullptr &&
|
||||
end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) {
|
||||
reporter_->Corruption(static_cast<size_t>(bytes), reason);
|
||||
}
|
||||
}
|
||||
|
||||
unsigned int Reader::ReadPhysicalRecord(Slice* result) {
|
||||
while (true) {
|
||||
if (buffer_.size() < kHeaderSize) {
|
||||
if (!eof_) {
|
||||
// Last read was a full read, so this is a trailer to skip
|
||||
buffer_.clear();
|
||||
Status status = file_->Read(kBlockSize, &buffer_, backing_store_);
|
||||
end_of_buffer_offset_ += buffer_.size();
|
||||
if (!status.ok()) {
|
||||
buffer_.clear();
|
||||
ReportDrop(kBlockSize, status);
|
||||
eof_ = true;
|
||||
return kEof;
|
||||
} else if (buffer_.size() < kBlockSize) {
|
||||
eof_ = true;
|
||||
}
|
||||
continue;
|
||||
} else {
|
||||
// Note that if buffer_ is non-empty, we have a truncated header at the
|
||||
// end of the file, which can be caused by the writer crashing in the
|
||||
// middle of writing the header. Instead of considering this an error,
|
||||
// just report EOF.
|
||||
buffer_.clear();
|
||||
return kEof;
|
||||
}
|
||||
}
|
||||
|
||||
// Parse the header
|
||||
const char* header = buffer_.data();
|
||||
const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
|
||||
const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
|
||||
const unsigned int type = header[6];
|
||||
const uint32_t length = a | (b << 8);
|
||||
if (kHeaderSize + length > buffer_.size()) {
|
||||
size_t drop_size = buffer_.size();
|
||||
buffer_.clear();
|
||||
if (!eof_) {
|
||||
ReportCorruption(drop_size, "bad record length");
|
||||
return kBadRecord;
|
||||
}
|
||||
// If the end of the file has been reached without reading |length| bytes
|
||||
// of payload, assume the writer died in the middle of writing the record.
|
||||
// Don't report a corruption.
|
||||
return kEof;
|
||||
}
|
||||
|
||||
if (type == kZeroType && length == 0) {
|
||||
// Skip zero length record without reporting any drops since
|
||||
// such records are produced by the mmap based writing code in
|
||||
// env_posix.cc that preallocates file regions.
|
||||
buffer_.clear();
|
||||
return kBadRecord;
|
||||
}
|
||||
|
||||
// Check crc
|
||||
if (checksum_) {
|
||||
uint32_t expected_crc = crc32c::Unmask(DecodeFixed32(header));
|
||||
uint32_t actual_crc = crc32c::Value(header + 6, 1 + length);
|
||||
if (actual_crc != expected_crc) {
|
||||
// Drop the rest of the buffer since "length" itself may have
|
||||
// been corrupted and if we trust it, we could find some
|
||||
// fragment of a real log record that just happens to look
|
||||
// like a valid log record.
|
||||
size_t drop_size = buffer_.size();
|
||||
buffer_.clear();
|
||||
ReportCorruption(drop_size, "checksum mismatch");
|
||||
return kBadRecord;
|
||||
}
|
||||
}
|
||||
|
||||
buffer_.remove_prefix(kHeaderSize + length);
|
||||
|
||||
// Skip physical record that started before initial_offset_
|
||||
if (end_of_buffer_offset_ - buffer_.size() - kHeaderSize - length <
|
||||
initial_offset_) {
|
||||
result->clear();
|
||||
return kBadRecord;
|
||||
}
|
||||
|
||||
*result = Slice(header + kHeaderSize, length);
|
||||
return type;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace log
|
||||
} // namespace leveldb
|
||||
Generated
+112
@@ -0,0 +1,112 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_READER_H_
|
||||
#define STORAGE_LEVELDB_DB_LOG_READER_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "db/log_format.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class SequentialFile;
|
||||
|
||||
namespace log {
|
||||
|
||||
class Reader {
|
||||
public:
|
||||
// Interface for reporting errors.
|
||||
class Reporter {
|
||||
public:
|
||||
virtual ~Reporter();
|
||||
|
||||
// Some corruption was detected. "size" is the approximate number
|
||||
// of bytes dropped due to the corruption.
|
||||
virtual void Corruption(size_t bytes, const Status& status) = 0;
|
||||
};
|
||||
|
||||
// Create a reader that will return log records from "*file".
|
||||
// "*file" must remain live while this Reader is in use.
|
||||
//
|
||||
// If "reporter" is non-null, it is notified whenever some data is
|
||||
// dropped due to a detected corruption. "*reporter" must remain
|
||||
// live while this Reader is in use.
|
||||
//
|
||||
// If "checksum" is true, verify checksums if available.
|
||||
//
|
||||
// The Reader will start reading at the first record located at physical
|
||||
// position >= initial_offset within the file.
|
||||
Reader(SequentialFile* file, Reporter* reporter, bool checksum,
|
||||
uint64_t initial_offset);
|
||||
|
||||
Reader(const Reader&) = delete;
|
||||
Reader& operator=(const Reader&) = delete;
|
||||
|
||||
~Reader();
|
||||
|
||||
// Read the next record into *record. Returns true if read
|
||||
// successfully, false if we hit end of the input. May use
|
||||
// "*scratch" as temporary storage. The contents filled in *record
|
||||
// will only be valid until the next mutating operation on this
|
||||
// reader or the next mutation to *scratch.
|
||||
bool ReadRecord(Slice* record, std::string* scratch);
|
||||
|
||||
// Returns the physical offset of the last record returned by ReadRecord.
|
||||
//
|
||||
// Undefined before the first call to ReadRecord.
|
||||
uint64_t LastRecordOffset();
|
||||
|
||||
private:
|
||||
// Extend record types with the following special values
|
||||
enum {
|
||||
kEof = kMaxRecordType + 1,
|
||||
// Returned whenever we find an invalid physical record.
|
||||
// Currently there are three situations in which this happens:
|
||||
// * The record has an invalid CRC (ReadPhysicalRecord reports a drop)
|
||||
// * The record is a 0-length record (No drop is reported)
|
||||
// * The record is below constructor's initial_offset (No drop is reported)
|
||||
kBadRecord = kMaxRecordType + 2
|
||||
};
|
||||
|
||||
// Skips all blocks that are completely before "initial_offset_".
|
||||
//
|
||||
// Returns true on success. Handles reporting.
|
||||
bool SkipToInitialBlock();
|
||||
|
||||
// Return type, or one of the preceding special values
|
||||
unsigned int ReadPhysicalRecord(Slice* result);
|
||||
|
||||
// Reports dropped bytes to the reporter.
|
||||
// buffer_ must be updated to remove the dropped bytes prior to invocation.
|
||||
void ReportCorruption(uint64_t bytes, const char* reason);
|
||||
void ReportDrop(uint64_t bytes, const Status& reason);
|
||||
|
||||
SequentialFile* const file_;
|
||||
Reporter* const reporter_;
|
||||
bool const checksum_;
|
||||
char* const backing_store_;
|
||||
Slice buffer_;
|
||||
bool eof_; // Last Read() indicated EOF by returning < kBlockSize
|
||||
|
||||
// Offset of the last record returned by ReadRecord.
|
||||
uint64_t last_record_offset_;
|
||||
// Offset of the first location past the end of buffer_.
|
||||
uint64_t end_of_buffer_offset_;
|
||||
|
||||
// Offset at which to start looking for the first record to return
|
||||
uint64_t const initial_offset_;
|
||||
|
||||
// True if we are resynchronizing after a seek (initial_offset_ > 0). In
|
||||
// particular, a run of kMiddleType and kLastType records can be silently
|
||||
// skipped in this mode
|
||||
bool resyncing_;
|
||||
};
|
||||
|
||||
} // namespace log
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_LOG_READER_H_
|
||||
Generated
+111
@@ -0,0 +1,111 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/log_writer.h"
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "leveldb/env.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/crc32c.h"
|
||||
|
||||
namespace leveldb {
|
||||
namespace log {
|
||||
|
||||
static void InitTypeCrc(uint32_t* type_crc) {
|
||||
for (int i = 0; i <= kMaxRecordType; i++) {
|
||||
char t = static_cast<char>(i);
|
||||
type_crc[i] = crc32c::Value(&t, 1);
|
||||
}
|
||||
}
|
||||
|
||||
Writer::Writer(WritableFile* dest) : dest_(dest), block_offset_(0) {
|
||||
InitTypeCrc(type_crc_);
|
||||
}
|
||||
|
||||
Writer::Writer(WritableFile* dest, uint64_t dest_length)
|
||||
: dest_(dest), block_offset_(dest_length % kBlockSize) {
|
||||
InitTypeCrc(type_crc_);
|
||||
}
|
||||
|
||||
Writer::~Writer() {}
|
||||
|
||||
Status Writer::AddRecord(const Slice& slice) {
|
||||
const char* ptr = slice.data();
|
||||
size_t left = slice.size();
|
||||
|
||||
// Fragment the record if necessary and emit it. Note that if slice
|
||||
// is empty, we still want to iterate once to emit a single
|
||||
// zero-length record
|
||||
Status s;
|
||||
bool begin = true;
|
||||
do {
|
||||
const int leftover = kBlockSize - block_offset_;
|
||||
assert(leftover >= 0);
|
||||
if (leftover < kHeaderSize) {
|
||||
// Switch to a new block
|
||||
if (leftover > 0) {
|
||||
// Fill the trailer (literal below relies on kHeaderSize being 7)
|
||||
static_assert(kHeaderSize == 7, "");
|
||||
dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover));
|
||||
}
|
||||
block_offset_ = 0;
|
||||
}
|
||||
|
||||
// Invariant: we never leave < kHeaderSize bytes in a block.
|
||||
assert(kBlockSize - block_offset_ - kHeaderSize >= 0);
|
||||
|
||||
const size_t avail = kBlockSize - block_offset_ - kHeaderSize;
|
||||
const size_t fragment_length = (left < avail) ? left : avail;
|
||||
|
||||
RecordType type;
|
||||
const bool end = (left == fragment_length);
|
||||
if (begin && end) {
|
||||
type = kFullType;
|
||||
} else if (begin) {
|
||||
type = kFirstType;
|
||||
} else if (end) {
|
||||
type = kLastType;
|
||||
} else {
|
||||
type = kMiddleType;
|
||||
}
|
||||
|
||||
s = EmitPhysicalRecord(type, ptr, fragment_length);
|
||||
ptr += fragment_length;
|
||||
left -= fragment_length;
|
||||
begin = false;
|
||||
} while (s.ok() && left > 0);
|
||||
return s;
|
||||
}
|
||||
|
||||
Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr,
|
||||
size_t length) {
|
||||
assert(length <= 0xffff); // Must fit in two bytes
|
||||
assert(block_offset_ + kHeaderSize + length <= kBlockSize);
|
||||
|
||||
// Format the header
|
||||
char buf[kHeaderSize];
|
||||
buf[4] = static_cast<char>(length & 0xff);
|
||||
buf[5] = static_cast<char>(length >> 8);
|
||||
buf[6] = static_cast<char>(t);
|
||||
|
||||
// Compute the crc of the record type and the payload.
|
||||
uint32_t crc = crc32c::Extend(type_crc_[t], ptr, length);
|
||||
crc = crc32c::Mask(crc); // Adjust for storage
|
||||
EncodeFixed32(buf, crc);
|
||||
|
||||
// Write the header and the payload
|
||||
Status s = dest_->Append(Slice(buf, kHeaderSize));
|
||||
if (s.ok()) {
|
||||
s = dest_->Append(Slice(ptr, length));
|
||||
if (s.ok()) {
|
||||
s = dest_->Flush();
|
||||
}
|
||||
}
|
||||
block_offset_ += kHeaderSize + length;
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace log
|
||||
} // namespace leveldb
|
||||
Generated
+54
@@ -0,0 +1,54 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_
|
||||
#define STORAGE_LEVELDB_DB_LOG_WRITER_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "db/log_format.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class WritableFile;
|
||||
|
||||
namespace log {
|
||||
|
||||
class Writer {
|
||||
public:
|
||||
// Create a writer that will append data to "*dest".
|
||||
// "*dest" must be initially empty.
|
||||
// "*dest" must remain live while this Writer is in use.
|
||||
explicit Writer(WritableFile* dest);
|
||||
|
||||
// Create a writer that will append data to "*dest".
|
||||
// "*dest" must have initial length "dest_length".
|
||||
// "*dest" must remain live while this Writer is in use.
|
||||
Writer(WritableFile* dest, uint64_t dest_length);
|
||||
|
||||
Writer(const Writer&) = delete;
|
||||
Writer& operator=(const Writer&) = delete;
|
||||
|
||||
~Writer();
|
||||
|
||||
Status AddRecord(const Slice& slice);
|
||||
|
||||
private:
|
||||
Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
|
||||
|
||||
WritableFile* dest_;
|
||||
int block_offset_; // Current offset in block
|
||||
|
||||
// crc32c values for all supported record types. These are
|
||||
// pre-computed to reduce the overhead of computing the crc of the
|
||||
// record type stored in the header.
|
||||
uint32_t type_crc_[kMaxRecordType + 1];
|
||||
};
|
||||
|
||||
} // namespace log
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_
|
||||
Generated
+136
@@ -0,0 +1,136 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/memtable.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
static Slice GetLengthPrefixedSlice(const char* data) {
|
||||
uint32_t len;
|
||||
const char* p = data;
|
||||
p = GetVarint32Ptr(p, p + 5, &len); // +5: we assume "p" is not corrupted
|
||||
return Slice(p, len);
|
||||
}
|
||||
|
||||
MemTable::MemTable(const InternalKeyComparator& comparator)
|
||||
: comparator_(comparator), refs_(0), table_(comparator_, &arena_) {}
|
||||
|
||||
MemTable::~MemTable() { assert(refs_ == 0); }
|
||||
|
||||
size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); }
|
||||
|
||||
int MemTable::KeyComparator::operator()(const char* aptr,
|
||||
const char* bptr) const {
|
||||
// Internal keys are encoded as length-prefixed strings.
|
||||
Slice a = GetLengthPrefixedSlice(aptr);
|
||||
Slice b = GetLengthPrefixedSlice(bptr);
|
||||
return comparator.Compare(a, b);
|
||||
}
|
||||
|
||||
// Encode a suitable internal key target for "target" and return it.
|
||||
// Uses *scratch as scratch space, and the returned pointer will point
|
||||
// into this scratch space.
|
||||
static const char* EncodeKey(std::string* scratch, const Slice& target) {
|
||||
scratch->clear();
|
||||
PutVarint32(scratch, target.size());
|
||||
scratch->append(target.data(), target.size());
|
||||
return scratch->data();
|
||||
}
|
||||
|
||||
class MemTableIterator : public Iterator {
|
||||
public:
|
||||
explicit MemTableIterator(MemTable::Table* table) : iter_(table) {}
|
||||
|
||||
virtual bool Valid() const { return iter_.Valid(); }
|
||||
virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); }
|
||||
virtual void SeekToFirst() { iter_.SeekToFirst(); }
|
||||
virtual void SeekToLast() { iter_.SeekToLast(); }
|
||||
virtual void Next() { iter_.Next(); }
|
||||
virtual void Prev() { iter_.Prev(); }
|
||||
virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); }
|
||||
virtual Slice value() const {
|
||||
Slice key_slice = GetLengthPrefixedSlice(iter_.key());
|
||||
return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
|
||||
}
|
||||
|
||||
virtual Status status() const { return Status::OK(); }
|
||||
|
||||
private:
|
||||
MemTable::Table::Iterator iter_;
|
||||
std::string tmp_; // For passing to EncodeKey
|
||||
|
||||
// No copying allowed
|
||||
MemTableIterator(const MemTableIterator&);
|
||||
void operator=(const MemTableIterator&);
|
||||
};
|
||||
|
||||
Iterator* MemTable::NewIterator() { return new MemTableIterator(&table_); }
|
||||
|
||||
void MemTable::Add(SequenceNumber s, ValueType type, const Slice& key,
|
||||
const Slice& value) {
|
||||
// Format of an entry is concatenation of:
|
||||
// key_size : varint32 of internal_key.size()
|
||||
// key bytes : char[internal_key.size()]
|
||||
// value_size : varint32 of value.size()
|
||||
// value bytes : char[value.size()]
|
||||
size_t key_size = key.size();
|
||||
size_t val_size = value.size();
|
||||
size_t internal_key_size = key_size + 8;
|
||||
const size_t encoded_len = VarintLength(internal_key_size) +
|
||||
internal_key_size + VarintLength(val_size) +
|
||||
val_size;
|
||||
char* buf = arena_.Allocate(encoded_len);
|
||||
char* p = EncodeVarint32(buf, internal_key_size);
|
||||
memcpy(p, key.data(), key_size);
|
||||
p += key_size;
|
||||
EncodeFixed64(p, (s << 8) | type);
|
||||
p += 8;
|
||||
p = EncodeVarint32(p, val_size);
|
||||
memcpy(p, value.data(), val_size);
|
||||
assert(p + val_size == buf + encoded_len);
|
||||
table_.Insert(buf);
|
||||
}
|
||||
|
||||
bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
|
||||
Slice memkey = key.memtable_key();
|
||||
Table::Iterator iter(&table_);
|
||||
iter.Seek(memkey.data());
|
||||
if (iter.Valid()) {
|
||||
// entry format is:
|
||||
// klength varint32
|
||||
// userkey char[klength]
|
||||
// tag uint64
|
||||
// vlength varint32
|
||||
// value char[vlength]
|
||||
// Check that it belongs to same user key. We do not check the
|
||||
// sequence number since the Seek() call above should have skipped
|
||||
// all entries with overly large sequence numbers.
|
||||
const char* entry = iter.key();
|
||||
uint32_t key_length;
|
||||
const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
|
||||
if (comparator_.comparator.user_comparator()->Compare(
|
||||
Slice(key_ptr, key_length - 8), key.user_key()) == 0) {
|
||||
// Correct user key
|
||||
const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
|
||||
switch (static_cast<ValueType>(tag & 0xff)) {
|
||||
case kTypeValue: {
|
||||
Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
|
||||
value->assign(v.data(), v.size());
|
||||
return true;
|
||||
}
|
||||
case kTypeDeletion:
|
||||
*s = Status::NotFound(Slice());
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
Generated
+87
@@ -0,0 +1,87 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_MEMTABLE_H_
|
||||
#define STORAGE_LEVELDB_DB_MEMTABLE_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/skiplist.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "util/arena.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class InternalKeyComparator;
|
||||
class MemTableIterator;
|
||||
|
||||
class MemTable {
|
||||
public:
|
||||
// MemTables are reference counted. The initial reference count
|
||||
// is zero and the caller must call Ref() at least once.
|
||||
explicit MemTable(const InternalKeyComparator& comparator);
|
||||
|
||||
MemTable(const MemTable&) = delete;
|
||||
MemTable& operator=(const MemTable&) = delete;
|
||||
|
||||
// Increase reference count.
|
||||
void Ref() { ++refs_; }
|
||||
|
||||
// Drop reference count. Delete if no more references exist.
|
||||
void Unref() {
|
||||
--refs_;
|
||||
assert(refs_ >= 0);
|
||||
if (refs_ <= 0) {
|
||||
delete this;
|
||||
}
|
||||
}
|
||||
|
||||
// Returns an estimate of the number of bytes of data in use by this
|
||||
// data structure. It is safe to call when MemTable is being modified.
|
||||
size_t ApproximateMemoryUsage();
|
||||
|
||||
// Return an iterator that yields the contents of the memtable.
|
||||
//
|
||||
// The caller must ensure that the underlying MemTable remains live
|
||||
// while the returned iterator is live. The keys returned by this
|
||||
// iterator are internal keys encoded by AppendInternalKey in the
|
||||
// db/format.{h,cc} module.
|
||||
Iterator* NewIterator();
|
||||
|
||||
// Add an entry into memtable that maps key to value at the
|
||||
// specified sequence number and with the specified type.
|
||||
// Typically value will be empty if type==kTypeDeletion.
|
||||
void Add(SequenceNumber seq, ValueType type, const Slice& key,
|
||||
const Slice& value);
|
||||
|
||||
// If memtable contains a value for key, store it in *value and return true.
|
||||
// If memtable contains a deletion for key, store a NotFound() error
|
||||
// in *status and return true.
|
||||
// Else, return false.
|
||||
bool Get(const LookupKey& key, std::string* value, Status* s);
|
||||
|
||||
private:
|
||||
friend class MemTableIterator;
|
||||
friend class MemTableBackwardIterator;
|
||||
|
||||
struct KeyComparator {
|
||||
const InternalKeyComparator comparator;
|
||||
explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {}
|
||||
int operator()(const char* a, const char* b) const;
|
||||
};
|
||||
|
||||
typedef SkipList<const char*, KeyComparator> Table;
|
||||
|
||||
~MemTable(); // Private since only Unref() should be used to delete it
|
||||
|
||||
KeyComparator comparator_;
|
||||
int refs_;
|
||||
Arena arena_;
|
||||
Table table_;
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_MEMTABLE_H_
|
||||
Generated
+450
@@ -0,0 +1,450 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// We recover the contents of the descriptor from the other files we find.
|
||||
// (1) Any log files are first converted to tables
|
||||
// (2) We scan every table to compute
|
||||
// (a) smallest/largest for the table
|
||||
// (b) largest sequence number in the table
|
||||
// (3) We generate descriptor contents:
|
||||
// - log number is set to zero
|
||||
// - next-file-number is set to 1 + largest file number we found
|
||||
// - last-sequence-number is set to largest sequence# found across
|
||||
// all tables (see 2c)
|
||||
// - compaction pointers are cleared
|
||||
// - every table file is added at level 0
|
||||
//
|
||||
// Possible optimization 1:
|
||||
// (a) Compute total size and use to pick appropriate max-level M
|
||||
// (b) Sort tables by largest sequence# in the table
|
||||
// (c) For each table: if it overlaps earlier table, place in level-0,
|
||||
// else place in level-M.
|
||||
// Possible optimization 2:
|
||||
// Store per-table metadata (smallest, largest, largest-seq#, ...)
|
||||
// in the table's meta section to speed up ScanTable.
|
||||
|
||||
#include "db/builder.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/log_reader.h"
|
||||
#include "db/log_writer.h"
|
||||
#include "db/memtable.h"
|
||||
#include "db/table_cache.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
namespace {
|
||||
|
||||
class Repairer {
|
||||
public:
|
||||
Repairer(const std::string& dbname, const Options& options)
|
||||
: dbname_(dbname),
|
||||
env_(options.env),
|
||||
icmp_(options.comparator),
|
||||
ipolicy_(options.filter_policy),
|
||||
options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
|
||||
owns_info_log_(options_.info_log != options.info_log),
|
||||
owns_cache_(options_.block_cache != options.block_cache),
|
||||
next_file_number_(1) {
|
||||
// TableCache can be small since we expect each table to be opened once.
|
||||
table_cache_ = new TableCache(dbname_, options_, 10);
|
||||
}
|
||||
|
||||
~Repairer() {
|
||||
delete table_cache_;
|
||||
if (owns_info_log_) {
|
||||
delete options_.info_log;
|
||||
}
|
||||
if (owns_cache_) {
|
||||
delete options_.block_cache;
|
||||
}
|
||||
}
|
||||
|
||||
Status Run() {
|
||||
Status status = FindFiles();
|
||||
if (status.ok()) {
|
||||
ConvertLogFilesToTables();
|
||||
ExtractMetaData();
|
||||
status = WriteDescriptor();
|
||||
}
|
||||
if (status.ok()) {
|
||||
unsigned long long bytes = 0;
|
||||
for (size_t i = 0; i < tables_.size(); i++) {
|
||||
bytes += tables_[i].meta.file_size;
|
||||
}
|
||||
Log(options_.info_log,
|
||||
"**** Repaired leveldb %s; "
|
||||
"recovered %d files; %llu bytes. "
|
||||
"Some data may have been lost. "
|
||||
"****",
|
||||
dbname_.c_str(), static_cast<int>(tables_.size()), bytes);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
private:
|
||||
struct TableInfo {
|
||||
FileMetaData meta;
|
||||
SequenceNumber max_sequence;
|
||||
};
|
||||
|
||||
Status FindFiles() {
|
||||
std::vector<std::string> filenames;
|
||||
Status status = env_->GetChildren(dbname_, &filenames);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
if (filenames.empty()) {
|
||||
return Status::IOError(dbname_, "repair found no files");
|
||||
}
|
||||
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
if (ParseFileName(filenames[i], &number, &type)) {
|
||||
if (type == kDescriptorFile) {
|
||||
manifests_.push_back(filenames[i]);
|
||||
} else {
|
||||
if (number + 1 > next_file_number_) {
|
||||
next_file_number_ = number + 1;
|
||||
}
|
||||
if (type == kLogFile) {
|
||||
logs_.push_back(number);
|
||||
} else if (type == kTableFile) {
|
||||
table_numbers_.push_back(number);
|
||||
} else {
|
||||
// Ignore other files
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
void ConvertLogFilesToTables() {
|
||||
for (size_t i = 0; i < logs_.size(); i++) {
|
||||
std::string logname = LogFileName(dbname_, logs_[i]);
|
||||
Status status = ConvertLogToTable(logs_[i]);
|
||||
if (!status.ok()) {
|
||||
Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
|
||||
(unsigned long long)logs_[i], status.ToString().c_str());
|
||||
}
|
||||
ArchiveFile(logname);
|
||||
}
|
||||
}
|
||||
|
||||
Status ConvertLogToTable(uint64_t log) {
|
||||
struct LogReporter : public log::Reader::Reporter {
|
||||
Env* env;
|
||||
Logger* info_log;
|
||||
uint64_t lognum;
|
||||
virtual void Corruption(size_t bytes, const Status& s) {
|
||||
// We print error messages for corruption, but continue repairing.
|
||||
Log(info_log, "Log #%llu: dropping %d bytes; %s",
|
||||
(unsigned long long)lognum, static_cast<int>(bytes),
|
||||
s.ToString().c_str());
|
||||
}
|
||||
};
|
||||
|
||||
// Open the log file
|
||||
std::string logname = LogFileName(dbname_, log);
|
||||
SequentialFile* lfile;
|
||||
Status status = env_->NewSequentialFile(logname, &lfile);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
|
||||
// Create the log reader.
|
||||
LogReporter reporter;
|
||||
reporter.env = env_;
|
||||
reporter.info_log = options_.info_log;
|
||||
reporter.lognum = log;
|
||||
// We intentionally make log::Reader do checksumming so that
|
||||
// corruptions cause entire commits to be skipped instead of
|
||||
// propagating bad information (like overly large sequence
|
||||
// numbers).
|
||||
log::Reader reader(lfile, &reporter, false /*do not checksum*/,
|
||||
0 /*initial_offset*/);
|
||||
|
||||
// Read all the records and add to a memtable
|
||||
std::string scratch;
|
||||
Slice record;
|
||||
WriteBatch batch;
|
||||
MemTable* mem = new MemTable(icmp_);
|
||||
mem->Ref();
|
||||
int counter = 0;
|
||||
while (reader.ReadRecord(&record, &scratch)) {
|
||||
if (record.size() < 12) {
|
||||
reporter.Corruption(record.size(),
|
||||
Status::Corruption("log record too small"));
|
||||
continue;
|
||||
}
|
||||
WriteBatchInternal::SetContents(&batch, record);
|
||||
status = WriteBatchInternal::InsertInto(&batch, mem);
|
||||
if (status.ok()) {
|
||||
counter += WriteBatchInternal::Count(&batch);
|
||||
} else {
|
||||
Log(options_.info_log, "Log #%llu: ignoring %s",
|
||||
(unsigned long long)log, status.ToString().c_str());
|
||||
status = Status::OK(); // Keep going with rest of file
|
||||
}
|
||||
}
|
||||
delete lfile;
|
||||
|
||||
// Do not record a version edit for this conversion to a Table
|
||||
// since ExtractMetaData() will also generate edits.
|
||||
FileMetaData meta;
|
||||
meta.number = next_file_number_++;
|
||||
Iterator* iter = mem->NewIterator();
|
||||
status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
|
||||
delete iter;
|
||||
mem->Unref();
|
||||
mem = nullptr;
|
||||
if (status.ok()) {
|
||||
if (meta.file_size > 0) {
|
||||
table_numbers_.push_back(meta.number);
|
||||
}
|
||||
}
|
||||
Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
|
||||
(unsigned long long)log, counter, (unsigned long long)meta.number,
|
||||
status.ToString().c_str());
|
||||
return status;
|
||||
}
|
||||
|
||||
void ExtractMetaData() {
|
||||
for (size_t i = 0; i < table_numbers_.size(); i++) {
|
||||
ScanTable(table_numbers_[i]);
|
||||
}
|
||||
}
|
||||
|
||||
Iterator* NewTableIterator(const FileMetaData& meta) {
|
||||
// Same as compaction iterators: if paranoid_checks are on, turn
|
||||
// on checksum verification.
|
||||
ReadOptions r;
|
||||
r.verify_checksums = options_.paranoid_checks;
|
||||
return table_cache_->NewIterator(r, meta.number, meta.file_size);
|
||||
}
|
||||
|
||||
void ScanTable(uint64_t number) {
|
||||
TableInfo t;
|
||||
t.meta.number = number;
|
||||
std::string fname = TableFileName(dbname_, number);
|
||||
Status status = env_->GetFileSize(fname, &t.meta.file_size);
|
||||
if (!status.ok()) {
|
||||
// Try alternate file name.
|
||||
fname = SSTTableFileName(dbname_, number);
|
||||
Status s2 = env_->GetFileSize(fname, &t.meta.file_size);
|
||||
if (s2.ok()) {
|
||||
status = Status::OK();
|
||||
}
|
||||
}
|
||||
if (!status.ok()) {
|
||||
ArchiveFile(TableFileName(dbname_, number));
|
||||
ArchiveFile(SSTTableFileName(dbname_, number));
|
||||
Log(options_.info_log, "Table #%llu: dropped: %s",
|
||||
(unsigned long long)t.meta.number, status.ToString().c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract metadata by scanning through table.
|
||||
int counter = 0;
|
||||
Iterator* iter = NewTableIterator(t.meta);
|
||||
bool empty = true;
|
||||
ParsedInternalKey parsed;
|
||||
t.max_sequence = 0;
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
Slice key = iter->key();
|
||||
if (!ParseInternalKey(key, &parsed)) {
|
||||
Log(options_.info_log, "Table #%llu: unparsable key %s",
|
||||
(unsigned long long)t.meta.number, EscapeString(key).c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
counter++;
|
||||
if (empty) {
|
||||
empty = false;
|
||||
t.meta.smallest.DecodeFrom(key);
|
||||
}
|
||||
t.meta.largest.DecodeFrom(key);
|
||||
if (parsed.sequence > t.max_sequence) {
|
||||
t.max_sequence = parsed.sequence;
|
||||
}
|
||||
}
|
||||
if (!iter->status().ok()) {
|
||||
status = iter->status();
|
||||
}
|
||||
delete iter;
|
||||
Log(options_.info_log, "Table #%llu: %d entries %s",
|
||||
(unsigned long long)t.meta.number, counter, status.ToString().c_str());
|
||||
|
||||
if (status.ok()) {
|
||||
tables_.push_back(t);
|
||||
} else {
|
||||
RepairTable(fname, t); // RepairTable archives input file.
|
||||
}
|
||||
}
|
||||
|
||||
void RepairTable(const std::string& src, TableInfo t) {
|
||||
// We will copy src contents to a new table and then rename the
|
||||
// new table over the source.
|
||||
|
||||
// Create builder.
|
||||
std::string copy = TableFileName(dbname_, next_file_number_++);
|
||||
WritableFile* file;
|
||||
Status s = env_->NewWritableFile(copy, &file);
|
||||
if (!s.ok()) {
|
||||
return;
|
||||
}
|
||||
TableBuilder* builder = new TableBuilder(options_, file);
|
||||
|
||||
// Copy data.
|
||||
Iterator* iter = NewTableIterator(t.meta);
|
||||
int counter = 0;
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
builder->Add(iter->key(), iter->value());
|
||||
counter++;
|
||||
}
|
||||
delete iter;
|
||||
|
||||
ArchiveFile(src);
|
||||
if (counter == 0) {
|
||||
builder->Abandon(); // Nothing to save
|
||||
} else {
|
||||
s = builder->Finish();
|
||||
if (s.ok()) {
|
||||
t.meta.file_size = builder->FileSize();
|
||||
}
|
||||
}
|
||||
delete builder;
|
||||
builder = nullptr;
|
||||
|
||||
if (s.ok()) {
|
||||
s = file->Close();
|
||||
}
|
||||
delete file;
|
||||
file = nullptr;
|
||||
|
||||
if (counter > 0 && s.ok()) {
|
||||
std::string orig = TableFileName(dbname_, t.meta.number);
|
||||
s = env_->RenameFile(copy, orig);
|
||||
if (s.ok()) {
|
||||
Log(options_.info_log, "Table #%llu: %d entries repaired",
|
||||
(unsigned long long)t.meta.number, counter);
|
||||
tables_.push_back(t);
|
||||
}
|
||||
}
|
||||
if (!s.ok()) {
|
||||
env_->DeleteFile(copy);
|
||||
}
|
||||
}
|
||||
|
||||
Status WriteDescriptor() {
|
||||
std::string tmp = TempFileName(dbname_, 1);
|
||||
WritableFile* file;
|
||||
Status status = env_->NewWritableFile(tmp, &file);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
|
||||
SequenceNumber max_sequence = 0;
|
||||
for (size_t i = 0; i < tables_.size(); i++) {
|
||||
if (max_sequence < tables_[i].max_sequence) {
|
||||
max_sequence = tables_[i].max_sequence;
|
||||
}
|
||||
}
|
||||
|
||||
edit_.SetComparatorName(icmp_.user_comparator()->Name());
|
||||
edit_.SetLogNumber(0);
|
||||
edit_.SetNextFile(next_file_number_);
|
||||
edit_.SetLastSequence(max_sequence);
|
||||
|
||||
for (size_t i = 0; i < tables_.size(); i++) {
|
||||
// TODO(opt): separate out into multiple levels
|
||||
const TableInfo& t = tables_[i];
|
||||
edit_.AddFile(0, t.meta.number, t.meta.file_size, t.meta.smallest,
|
||||
t.meta.largest);
|
||||
}
|
||||
|
||||
// fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
|
||||
{
|
||||
log::Writer log(file);
|
||||
std::string record;
|
||||
edit_.EncodeTo(&record);
|
||||
status = log.AddRecord(record);
|
||||
}
|
||||
if (status.ok()) {
|
||||
status = file->Close();
|
||||
}
|
||||
delete file;
|
||||
file = nullptr;
|
||||
|
||||
if (!status.ok()) {
|
||||
env_->DeleteFile(tmp);
|
||||
} else {
|
||||
// Discard older manifests
|
||||
for (size_t i = 0; i < manifests_.size(); i++) {
|
||||
ArchiveFile(dbname_ + "/" + manifests_[i]);
|
||||
}
|
||||
|
||||
// Install new manifest
|
||||
status = env_->RenameFile(tmp, DescriptorFileName(dbname_, 1));
|
||||
if (status.ok()) {
|
||||
status = SetCurrentFile(env_, dbname_, 1);
|
||||
} else {
|
||||
env_->DeleteFile(tmp);
|
||||
}
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
void ArchiveFile(const std::string& fname) {
|
||||
// Move into another directory. E.g., for
|
||||
// dir/foo
|
||||
// rename to
|
||||
// dir/lost/foo
|
||||
const char* slash = strrchr(fname.c_str(), '/');
|
||||
std::string new_dir;
|
||||
if (slash != nullptr) {
|
||||
new_dir.assign(fname.data(), slash - fname.data());
|
||||
}
|
||||
new_dir.append("/lost");
|
||||
env_->CreateDir(new_dir); // Ignore error
|
||||
std::string new_file = new_dir;
|
||||
new_file.append("/");
|
||||
new_file.append((slash == nullptr) ? fname.c_str() : slash + 1);
|
||||
Status s = env_->RenameFile(fname, new_file);
|
||||
Log(options_.info_log, "Archiving %s: %s\n", fname.c_str(),
|
||||
s.ToString().c_str());
|
||||
}
|
||||
|
||||
const std::string dbname_;
|
||||
Env* const env_;
|
||||
InternalKeyComparator const icmp_;
|
||||
InternalFilterPolicy const ipolicy_;
|
||||
const Options options_;
|
||||
bool owns_info_log_;
|
||||
bool owns_cache_;
|
||||
TableCache* table_cache_;
|
||||
VersionEdit edit_;
|
||||
|
||||
std::vector<std::string> manifests_;
|
||||
std::vector<uint64_t> table_numbers_;
|
||||
std::vector<uint64_t> logs_;
|
||||
std::vector<TableInfo> tables_;
|
||||
uint64_t next_file_number_;
|
||||
};
|
||||
} // namespace
|
||||
|
||||
Status RepairDB(const std::string& dbname, const Options& options) {
|
||||
Repairer repairer(dbname, options);
|
||||
return repairer.Run();
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
Generated
+382
@@ -0,0 +1,382 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
#define STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
|
||||
// Thread safety
|
||||
// -------------
|
||||
//
|
||||
// Writes require external synchronization, most likely a mutex.
|
||||
// Reads require a guarantee that the SkipList will not be destroyed
|
||||
// while the read is in progress. Apart from that, reads progress
|
||||
// without any internal locking or synchronization.
|
||||
//
|
||||
// Invariants:
|
||||
//
|
||||
// (1) Allocated nodes are never deleted until the SkipList is
|
||||
// destroyed. This is trivially guaranteed by the code since we
|
||||
// never delete any skip list nodes.
|
||||
//
|
||||
// (2) The contents of a Node except for the next/prev pointers are
|
||||
// immutable after the Node has been linked into the SkipList.
|
||||
// Only Insert() modifies the list, and it is careful to initialize
|
||||
// a node and use release-stores to publish the nodes in one or
|
||||
// more lists.
|
||||
//
|
||||
// ... prev vs. next pointer ordering ...
|
||||
|
||||
#include <atomic>
|
||||
#include <cassert>
|
||||
#include <cstdlib>
|
||||
|
||||
#include "util/arena.h"
|
||||
#include "util/random.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class Arena;
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
class SkipList {
|
||||
private:
|
||||
struct Node;
|
||||
|
||||
public:
|
||||
// Create a new SkipList object that will use "cmp" for comparing keys,
|
||||
// and will allocate memory using "*arena". Objects allocated in the arena
|
||||
// must remain allocated for the lifetime of the skiplist object.
|
||||
explicit SkipList(Comparator cmp, Arena* arena);
|
||||
|
||||
SkipList(const SkipList&) = delete;
|
||||
SkipList& operator=(const SkipList&) = delete;
|
||||
|
||||
// Insert key into the list.
|
||||
// REQUIRES: nothing that compares equal to key is currently in the list.
|
||||
void Insert(const Key& key);
|
||||
|
||||
// Returns true iff an entry that compares equal to key is in the list.
|
||||
bool Contains(const Key& key) const;
|
||||
|
||||
// Iteration over the contents of a skip list
|
||||
class Iterator {
|
||||
public:
|
||||
// Initialize an iterator over the specified list.
|
||||
// The returned iterator is not valid.
|
||||
explicit Iterator(const SkipList* list);
|
||||
|
||||
// Returns true iff the iterator is positioned at a valid node.
|
||||
bool Valid() const;
|
||||
|
||||
// Returns the key at the current position.
|
||||
// REQUIRES: Valid()
|
||||
const Key& key() const;
|
||||
|
||||
// Advances to the next position.
|
||||
// REQUIRES: Valid()
|
||||
void Next();
|
||||
|
||||
// Advances to the previous position.
|
||||
// REQUIRES: Valid()
|
||||
void Prev();
|
||||
|
||||
// Advance to the first entry with a key >= target
|
||||
void Seek(const Key& target);
|
||||
|
||||
// Position at the first entry in list.
|
||||
// Final state of iterator is Valid() iff list is not empty.
|
||||
void SeekToFirst();
|
||||
|
||||
// Position at the last entry in list.
|
||||
// Final state of iterator is Valid() iff list is not empty.
|
||||
void SeekToLast();
|
||||
|
||||
private:
|
||||
const SkipList* list_;
|
||||
Node* node_;
|
||||
// Intentionally copyable
|
||||
};
|
||||
|
||||
private:
|
||||
enum { kMaxHeight = 12 };
|
||||
|
||||
inline int GetMaxHeight() const {
|
||||
return max_height_.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
Node* NewNode(const Key& key, int height);
|
||||
int RandomHeight();
|
||||
bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); }
|
||||
|
||||
// Return true if key is greater than the data stored in "n"
|
||||
bool KeyIsAfterNode(const Key& key, Node* n) const;
|
||||
|
||||
// Return the earliest node that comes at or after key.
|
||||
// Return nullptr if there is no such node.
|
||||
//
|
||||
// If prev is non-null, fills prev[level] with pointer to previous
|
||||
// node at "level" for every level in [0..max_height_-1].
|
||||
Node* FindGreaterOrEqual(const Key& key, Node** prev) const;
|
||||
|
||||
// Return the latest node with a key < key.
|
||||
// Return head_ if there is no such node.
|
||||
Node* FindLessThan(const Key& key) const;
|
||||
|
||||
// Return the last node in the list.
|
||||
// Return head_ if list is empty.
|
||||
Node* FindLast() const;
|
||||
|
||||
// Immutable after construction
|
||||
Comparator const compare_;
|
||||
Arena* const arena_; // Arena used for allocations of nodes
|
||||
|
||||
Node* const head_;
|
||||
|
||||
// Modified only by Insert(). Read racily by readers, but stale
|
||||
// values are ok.
|
||||
std::atomic<int> max_height_; // Height of the entire list
|
||||
|
||||
// Read/written only by Insert().
|
||||
Random rnd_;
|
||||
};
|
||||
|
||||
// Implementation details follow
|
||||
template <typename Key, class Comparator>
|
||||
struct SkipList<Key, Comparator>::Node {
|
||||
explicit Node(const Key& k) : key(k) {}
|
||||
|
||||
Key const key;
|
||||
|
||||
// Accessors/mutators for links. Wrapped in methods so we can
|
||||
// add the appropriate barriers as necessary.
|
||||
Node* Next(int n) {
|
||||
assert(n >= 0);
|
||||
// Use an 'acquire load' so that we observe a fully initialized
|
||||
// version of the returned Node.
|
||||
return next_[n].load(std::memory_order_acquire);
|
||||
}
|
||||
void SetNext(int n, Node* x) {
|
||||
assert(n >= 0);
|
||||
// Use a 'release store' so that anybody who reads through this
|
||||
// pointer observes a fully initialized version of the inserted node.
|
||||
next_[n].store(x, std::memory_order_release);
|
||||
}
|
||||
|
||||
// No-barrier variants that can be safely used in a few locations.
|
||||
Node* NoBarrier_Next(int n) {
|
||||
assert(n >= 0);
|
||||
return next_[n].load(std::memory_order_relaxed);
|
||||
}
|
||||
void NoBarrier_SetNext(int n, Node* x) {
|
||||
assert(n >= 0);
|
||||
next_[n].store(x, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
private:
|
||||
// Array of length equal to the node height. next_[0] is lowest level link.
|
||||
std::atomic<Node*> next_[1];
|
||||
};
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::NewNode(
|
||||
const Key& key, int height) {
|
||||
char* const node_memory = arena_->AllocateAligned(
|
||||
sizeof(Node) + sizeof(std::atomic<Node*>) * (height - 1));
|
||||
return new (node_memory) Node(key);
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
inline SkipList<Key, Comparator>::Iterator::Iterator(const SkipList* list) {
|
||||
list_ = list;
|
||||
node_ = nullptr;
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
inline bool SkipList<Key, Comparator>::Iterator::Valid() const {
|
||||
return node_ != nullptr;
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
inline const Key& SkipList<Key, Comparator>::Iterator::key() const {
|
||||
assert(Valid());
|
||||
return node_->key;
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
inline void SkipList<Key, Comparator>::Iterator::Next() {
|
||||
assert(Valid());
|
||||
node_ = node_->Next(0);
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
inline void SkipList<Key, Comparator>::Iterator::Prev() {
|
||||
// Instead of using explicit "prev" links, we just search for the
|
||||
// last node that falls before key.
|
||||
assert(Valid());
|
||||
node_ = list_->FindLessThan(node_->key);
|
||||
if (node_ == list_->head_) {
|
||||
node_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
inline void SkipList<Key, Comparator>::Iterator::Seek(const Key& target) {
|
||||
node_ = list_->FindGreaterOrEqual(target, nullptr);
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
inline void SkipList<Key, Comparator>::Iterator::SeekToFirst() {
|
||||
node_ = list_->head_->Next(0);
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
inline void SkipList<Key, Comparator>::Iterator::SeekToLast() {
|
||||
node_ = list_->FindLast();
|
||||
if (node_ == list_->head_) {
|
||||
node_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
int SkipList<Key, Comparator>::RandomHeight() {
|
||||
// Increase height with probability 1 in kBranching
|
||||
static const unsigned int kBranching = 4;
|
||||
int height = 1;
|
||||
while (height < kMaxHeight && ((rnd_.Next() % kBranching) == 0)) {
|
||||
height++;
|
||||
}
|
||||
assert(height > 0);
|
||||
assert(height <= kMaxHeight);
|
||||
return height;
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
bool SkipList<Key, Comparator>::KeyIsAfterNode(const Key& key, Node* n) const {
|
||||
// null n is considered infinite
|
||||
return (n != nullptr) && (compare_(n->key, key) < 0);
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
typename SkipList<Key, Comparator>::Node*
|
||||
SkipList<Key, Comparator>::FindGreaterOrEqual(const Key& key,
|
||||
Node** prev) const {
|
||||
Node* x = head_;
|
||||
int level = GetMaxHeight() - 1;
|
||||
while (true) {
|
||||
Node* next = x->Next(level);
|
||||
if (KeyIsAfterNode(key, next)) {
|
||||
// Keep searching in this list
|
||||
x = next;
|
||||
} else {
|
||||
if (prev != nullptr) prev[level] = x;
|
||||
if (level == 0) {
|
||||
return next;
|
||||
} else {
|
||||
// Switch to next list
|
||||
level--;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
typename SkipList<Key, Comparator>::Node*
|
||||
SkipList<Key, Comparator>::FindLessThan(const Key& key) const {
|
||||
Node* x = head_;
|
||||
int level = GetMaxHeight() - 1;
|
||||
while (true) {
|
||||
assert(x == head_ || compare_(x->key, key) < 0);
|
||||
Node* next = x->Next(level);
|
||||
if (next == nullptr || compare_(next->key, key) >= 0) {
|
||||
if (level == 0) {
|
||||
return x;
|
||||
} else {
|
||||
// Switch to next list
|
||||
level--;
|
||||
}
|
||||
} else {
|
||||
x = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::FindLast()
|
||||
const {
|
||||
Node* x = head_;
|
||||
int level = GetMaxHeight() - 1;
|
||||
while (true) {
|
||||
Node* next = x->Next(level);
|
||||
if (next == nullptr) {
|
||||
if (level == 0) {
|
||||
return x;
|
||||
} else {
|
||||
// Switch to next list
|
||||
level--;
|
||||
}
|
||||
} else {
|
||||
x = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
SkipList<Key, Comparator>::SkipList(Comparator cmp, Arena* arena)
|
||||
: compare_(cmp),
|
||||
arena_(arena),
|
||||
head_(NewNode(0 /* any key will do */, kMaxHeight)),
|
||||
max_height_(1),
|
||||
rnd_(0xdeadbeef) {
|
||||
for (int i = 0; i < kMaxHeight; i++) {
|
||||
head_->SetNext(i, nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
void SkipList<Key, Comparator>::Insert(const Key& key) {
|
||||
// TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual()
|
||||
// here since Insert() is externally synchronized.
|
||||
Node* prev[kMaxHeight];
|
||||
Node* x = FindGreaterOrEqual(key, prev);
|
||||
|
||||
// Our data structure does not allow duplicate insertion
|
||||
assert(x == nullptr || !Equal(key, x->key));
|
||||
|
||||
int height = RandomHeight();
|
||||
if (height > GetMaxHeight()) {
|
||||
for (int i = GetMaxHeight(); i < height; i++) {
|
||||
prev[i] = head_;
|
||||
}
|
||||
// It is ok to mutate max_height_ without any synchronization
|
||||
// with concurrent readers. A concurrent reader that observes
|
||||
// the new value of max_height_ will see either the old value of
|
||||
// new level pointers from head_ (nullptr), or a new value set in
|
||||
// the loop below. In the former case the reader will
|
||||
// immediately drop to the next level since nullptr sorts after all
|
||||
// keys. In the latter case the reader will use the new node.
|
||||
max_height_.store(height, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
x = NewNode(key, height);
|
||||
for (int i = 0; i < height; i++) {
|
||||
// NoBarrier_SetNext() suffices since we will add a barrier when
|
||||
// we publish a pointer to "x" in prev[i].
|
||||
x->NoBarrier_SetNext(i, prev[i]->NoBarrier_Next(i));
|
||||
prev[i]->SetNext(i, x);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
bool SkipList<Key, Comparator>::Contains(const Key& key) const {
|
||||
Node* x = FindGreaterOrEqual(key, nullptr);
|
||||
if (x != nullptr && Equal(key, x->key)) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
Generated
+95
@@ -0,0 +1,95 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_
|
||||
#define STORAGE_LEVELDB_DB_SNAPSHOT_H_
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/db.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class SnapshotList;
|
||||
|
||||
// Snapshots are kept in a doubly-linked list in the DB.
|
||||
// Each SnapshotImpl corresponds to a particular sequence number.
|
||||
class SnapshotImpl : public Snapshot {
|
||||
public:
|
||||
SnapshotImpl(SequenceNumber sequence_number)
|
||||
: sequence_number_(sequence_number) {}
|
||||
|
||||
SequenceNumber sequence_number() const { return sequence_number_; }
|
||||
|
||||
private:
|
||||
friend class SnapshotList;
|
||||
|
||||
// SnapshotImpl is kept in a doubly-linked circular list. The SnapshotList
|
||||
// implementation operates on the next/previous fields direcly.
|
||||
SnapshotImpl* prev_;
|
||||
SnapshotImpl* next_;
|
||||
|
||||
const SequenceNumber sequence_number_;
|
||||
|
||||
#if !defined(NDEBUG)
|
||||
SnapshotList* list_ = nullptr;
|
||||
#endif // !defined(NDEBUG)
|
||||
};
|
||||
|
||||
class SnapshotList {
|
||||
public:
|
||||
SnapshotList() : head_(0) {
|
||||
head_.prev_ = &head_;
|
||||
head_.next_ = &head_;
|
||||
}
|
||||
|
||||
bool empty() const { return head_.next_ == &head_; }
|
||||
SnapshotImpl* oldest() const {
|
||||
assert(!empty());
|
||||
return head_.next_;
|
||||
}
|
||||
SnapshotImpl* newest() const {
|
||||
assert(!empty());
|
||||
return head_.prev_;
|
||||
}
|
||||
|
||||
// Creates a SnapshotImpl and appends it to the end of the list.
|
||||
SnapshotImpl* New(SequenceNumber sequence_number) {
|
||||
assert(empty() || newest()->sequence_number_ <= sequence_number);
|
||||
|
||||
SnapshotImpl* snapshot = new SnapshotImpl(sequence_number);
|
||||
|
||||
#if !defined(NDEBUG)
|
||||
snapshot->list_ = this;
|
||||
#endif // !defined(NDEBUG)
|
||||
snapshot->next_ = &head_;
|
||||
snapshot->prev_ = head_.prev_;
|
||||
snapshot->prev_->next_ = snapshot;
|
||||
snapshot->next_->prev_ = snapshot;
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
// Removes a SnapshotImpl from this list.
|
||||
//
|
||||
// The snapshot must have been created by calling New() on this list.
|
||||
//
|
||||
// The snapshot pointer should not be const, because its memory is
|
||||
// deallocated. However, that would force us to change DB::ReleaseSnapshot(),
|
||||
// which is in the API, and currently takes a const Snapshot.
|
||||
void Delete(const SnapshotImpl* snapshot) {
|
||||
#if !defined(NDEBUG)
|
||||
assert(snapshot->list_ == this);
|
||||
#endif // !defined(NDEBUG)
|
||||
snapshot->prev_->next_ = snapshot->next_;
|
||||
snapshot->next_->prev_ = snapshot->prev_;
|
||||
delete snapshot;
|
||||
}
|
||||
|
||||
private:
|
||||
// Dummy head of doubly-linked list of snapshots
|
||||
SnapshotImpl head_;
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_
|
||||
Generated
+120
@@ -0,0 +1,120 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/table_cache.h"
|
||||
|
||||
#include "db/filename.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
struct TableAndFile {
|
||||
RandomAccessFile* file;
|
||||
Table* table;
|
||||
};
|
||||
|
||||
static void DeleteEntry(const Slice& key, void* value) {
|
||||
TableAndFile* tf = reinterpret_cast<TableAndFile*>(value);
|
||||
delete tf->table;
|
||||
delete tf->file;
|
||||
delete tf;
|
||||
}
|
||||
|
||||
static void UnrefEntry(void* arg1, void* arg2) {
|
||||
Cache* cache = reinterpret_cast<Cache*>(arg1);
|
||||
Cache::Handle* h = reinterpret_cast<Cache::Handle*>(arg2);
|
||||
cache->Release(h);
|
||||
}
|
||||
|
||||
TableCache::TableCache(const std::string& dbname, const Options& options,
|
||||
int entries)
|
||||
: env_(options.env),
|
||||
dbname_(dbname),
|
||||
options_(options),
|
||||
cache_(NewLRUCache(entries)) {}
|
||||
|
||||
TableCache::~TableCache() { delete cache_; }
|
||||
|
||||
Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
|
||||
Cache::Handle** handle) {
|
||||
Status s;
|
||||
char buf[sizeof(file_number)];
|
||||
EncodeFixed64(buf, file_number);
|
||||
Slice key(buf, sizeof(buf));
|
||||
*handle = cache_->Lookup(key);
|
||||
if (*handle == nullptr) {
|
||||
std::string fname = TableFileName(dbname_, file_number);
|
||||
RandomAccessFile* file = nullptr;
|
||||
Table* table = nullptr;
|
||||
s = env_->NewRandomAccessFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
std::string old_fname = SSTTableFileName(dbname_, file_number);
|
||||
if (env_->NewRandomAccessFile(old_fname, &file).ok()) {
|
||||
s = Status::OK();
|
||||
}
|
||||
}
|
||||
if (s.ok()) {
|
||||
s = Table::Open(options_, file, file_size, &table);
|
||||
}
|
||||
|
||||
if (!s.ok()) {
|
||||
assert(table == nullptr);
|
||||
delete file;
|
||||
// We do not cache error results so that if the error is transient,
|
||||
// or somebody repairs the file, we recover automatically.
|
||||
} else {
|
||||
TableAndFile* tf = new TableAndFile;
|
||||
tf->file = file;
|
||||
tf->table = table;
|
||||
*handle = cache_->Insert(key, tf, 1, &DeleteEntry);
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Iterator* TableCache::NewIterator(const ReadOptions& options,
|
||||
uint64_t file_number, uint64_t file_size,
|
||||
Table** tableptr) {
|
||||
if (tableptr != nullptr) {
|
||||
*tableptr = nullptr;
|
||||
}
|
||||
|
||||
Cache::Handle* handle = nullptr;
|
||||
Status s = FindTable(file_number, file_size, &handle);
|
||||
if (!s.ok()) {
|
||||
return NewErrorIterator(s);
|
||||
}
|
||||
|
||||
Table* table = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
|
||||
Iterator* result = table->NewIterator(options);
|
||||
result->RegisterCleanup(&UnrefEntry, cache_, handle);
|
||||
if (tableptr != nullptr) {
|
||||
*tableptr = table;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Status TableCache::Get(const ReadOptions& options, uint64_t file_number,
|
||||
uint64_t file_size, const Slice& k, void* arg,
|
||||
void (*handle_result)(void*, const Slice&,
|
||||
const Slice&)) {
|
||||
Cache::Handle* handle = nullptr;
|
||||
Status s = FindTable(file_number, file_size, &handle);
|
||||
if (s.ok()) {
|
||||
Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
|
||||
s = t->InternalGet(options, k, arg, handle_result);
|
||||
cache_->Release(handle);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
void TableCache::Evict(uint64_t file_number) {
|
||||
char buf[sizeof(file_number)];
|
||||
EncodeFixed64(buf, file_number);
|
||||
cache_->Erase(Slice(buf, sizeof(buf)));
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
Generated
+58
@@ -0,0 +1,58 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// Thread-safe (provides internal synchronization)
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_
|
||||
#define STORAGE_LEVELDB_DB_TABLE_CACHE_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "port/port.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class Env;
|
||||
|
||||
class TableCache {
|
||||
public:
|
||||
TableCache(const std::string& dbname, const Options& options, int entries);
|
||||
~TableCache();
|
||||
|
||||
// Return an iterator for the specified file number (the corresponding
|
||||
// file length must be exactly "file_size" bytes). If "tableptr" is
|
||||
// non-null, also sets "*tableptr" to point to the Table object
|
||||
// underlying the returned iterator, or to nullptr if no Table object
|
||||
// underlies the returned iterator. The returned "*tableptr" object is owned
|
||||
// by the cache and should not be deleted, and is valid for as long as the
|
||||
// returned iterator is live.
|
||||
Iterator* NewIterator(const ReadOptions& options, uint64_t file_number,
|
||||
uint64_t file_size, Table** tableptr = nullptr);
|
||||
|
||||
// If a seek to internal key "k" in specified file finds an entry,
|
||||
// call (*handle_result)(arg, found_key, found_value).
|
||||
Status Get(const ReadOptions& options, uint64_t file_number,
|
||||
uint64_t file_size, const Slice& k, void* arg,
|
||||
void (*handle_result)(void*, const Slice&, const Slice&));
|
||||
|
||||
// Evict any entry for the specified file number
|
||||
void Evict(uint64_t file_number);
|
||||
|
||||
private:
|
||||
Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
|
||||
|
||||
Env* const env_;
|
||||
const std::string dbname_;
|
||||
const Options& options_;
|
||||
Cache* cache_;
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_
|
||||
Generated
+260
@@ -0,0 +1,260 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/version_edit.h"
|
||||
|
||||
#include "db/version_set.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
// Tag numbers for serialized VersionEdit. These numbers are written to
|
||||
// disk and should not be changed.
|
||||
enum Tag {
|
||||
kComparator = 1,
|
||||
kLogNumber = 2,
|
||||
kNextFileNumber = 3,
|
||||
kLastSequence = 4,
|
||||
kCompactPointer = 5,
|
||||
kDeletedFile = 6,
|
||||
kNewFile = 7,
|
||||
// 8 was used for large value refs
|
||||
kPrevLogNumber = 9
|
||||
};
|
||||
|
||||
void VersionEdit::Clear() {
|
||||
comparator_.clear();
|
||||
log_number_ = 0;
|
||||
prev_log_number_ = 0;
|
||||
last_sequence_ = 0;
|
||||
next_file_number_ = 0;
|
||||
has_comparator_ = false;
|
||||
has_log_number_ = false;
|
||||
has_prev_log_number_ = false;
|
||||
has_next_file_number_ = false;
|
||||
has_last_sequence_ = false;
|
||||
deleted_files_.clear();
|
||||
new_files_.clear();
|
||||
}
|
||||
|
||||
void VersionEdit::EncodeTo(std::string* dst) const {
|
||||
if (has_comparator_) {
|
||||
PutVarint32(dst, kComparator);
|
||||
PutLengthPrefixedSlice(dst, comparator_);
|
||||
}
|
||||
if (has_log_number_) {
|
||||
PutVarint32(dst, kLogNumber);
|
||||
PutVarint64(dst, log_number_);
|
||||
}
|
||||
if (has_prev_log_number_) {
|
||||
PutVarint32(dst, kPrevLogNumber);
|
||||
PutVarint64(dst, prev_log_number_);
|
||||
}
|
||||
if (has_next_file_number_) {
|
||||
PutVarint32(dst, kNextFileNumber);
|
||||
PutVarint64(dst, next_file_number_);
|
||||
}
|
||||
if (has_last_sequence_) {
|
||||
PutVarint32(dst, kLastSequence);
|
||||
PutVarint64(dst, last_sequence_);
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < compact_pointers_.size(); i++) {
|
||||
PutVarint32(dst, kCompactPointer);
|
||||
PutVarint32(dst, compact_pointers_[i].first); // level
|
||||
PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode());
|
||||
}
|
||||
|
||||
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
|
||||
iter != deleted_files_.end(); ++iter) {
|
||||
PutVarint32(dst, kDeletedFile);
|
||||
PutVarint32(dst, iter->first); // level
|
||||
PutVarint64(dst, iter->second); // file number
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < new_files_.size(); i++) {
|
||||
const FileMetaData& f = new_files_[i].second;
|
||||
PutVarint32(dst, kNewFile);
|
||||
PutVarint32(dst, new_files_[i].first); // level
|
||||
PutVarint64(dst, f.number);
|
||||
PutVarint64(dst, f.file_size);
|
||||
PutLengthPrefixedSlice(dst, f.smallest.Encode());
|
||||
PutLengthPrefixedSlice(dst, f.largest.Encode());
|
||||
}
|
||||
}
|
||||
|
||||
static bool GetInternalKey(Slice* input, InternalKey* dst) {
|
||||
Slice str;
|
||||
if (GetLengthPrefixedSlice(input, &str)) {
|
||||
dst->DecodeFrom(str);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static bool GetLevel(Slice* input, int* level) {
|
||||
uint32_t v;
|
||||
if (GetVarint32(input, &v) && v < config::kNumLevels) {
|
||||
*level = v;
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
Status VersionEdit::DecodeFrom(const Slice& src) {
|
||||
Clear();
|
||||
Slice input = src;
|
||||
const char* msg = nullptr;
|
||||
uint32_t tag;
|
||||
|
||||
// Temporary storage for parsing
|
||||
int level;
|
||||
uint64_t number;
|
||||
FileMetaData f;
|
||||
Slice str;
|
||||
InternalKey key;
|
||||
|
||||
while (msg == nullptr && GetVarint32(&input, &tag)) {
|
||||
switch (tag) {
|
||||
case kComparator:
|
||||
if (GetLengthPrefixedSlice(&input, &str)) {
|
||||
comparator_ = str.ToString();
|
||||
has_comparator_ = true;
|
||||
} else {
|
||||
msg = "comparator name";
|
||||
}
|
||||
break;
|
||||
|
||||
case kLogNumber:
|
||||
if (GetVarint64(&input, &log_number_)) {
|
||||
has_log_number_ = true;
|
||||
} else {
|
||||
msg = "log number";
|
||||
}
|
||||
break;
|
||||
|
||||
case kPrevLogNumber:
|
||||
if (GetVarint64(&input, &prev_log_number_)) {
|
||||
has_prev_log_number_ = true;
|
||||
} else {
|
||||
msg = "previous log number";
|
||||
}
|
||||
break;
|
||||
|
||||
case kNextFileNumber:
|
||||
if (GetVarint64(&input, &next_file_number_)) {
|
||||
has_next_file_number_ = true;
|
||||
} else {
|
||||
msg = "next file number";
|
||||
}
|
||||
break;
|
||||
|
||||
case kLastSequence:
|
||||
if (GetVarint64(&input, &last_sequence_)) {
|
||||
has_last_sequence_ = true;
|
||||
} else {
|
||||
msg = "last sequence number";
|
||||
}
|
||||
break;
|
||||
|
||||
case kCompactPointer:
|
||||
if (GetLevel(&input, &level) && GetInternalKey(&input, &key)) {
|
||||
compact_pointers_.push_back(std::make_pair(level, key));
|
||||
} else {
|
||||
msg = "compaction pointer";
|
||||
}
|
||||
break;
|
||||
|
||||
case kDeletedFile:
|
||||
if (GetLevel(&input, &level) && GetVarint64(&input, &number)) {
|
||||
deleted_files_.insert(std::make_pair(level, number));
|
||||
} else {
|
||||
msg = "deleted file";
|
||||
}
|
||||
break;
|
||||
|
||||
case kNewFile:
|
||||
if (GetLevel(&input, &level) && GetVarint64(&input, &f.number) &&
|
||||
GetVarint64(&input, &f.file_size) &&
|
||||
GetInternalKey(&input, &f.smallest) &&
|
||||
GetInternalKey(&input, &f.largest)) {
|
||||
new_files_.push_back(std::make_pair(level, f));
|
||||
} else {
|
||||
msg = "new-file entry";
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
msg = "unknown tag";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (msg == nullptr && !input.empty()) {
|
||||
msg = "invalid tag";
|
||||
}
|
||||
|
||||
Status result;
|
||||
if (msg != nullptr) {
|
||||
result = Status::Corruption("VersionEdit", msg);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string VersionEdit::DebugString() const {
|
||||
std::string r;
|
||||
r.append("VersionEdit {");
|
||||
if (has_comparator_) {
|
||||
r.append("\n Comparator: ");
|
||||
r.append(comparator_);
|
||||
}
|
||||
if (has_log_number_) {
|
||||
r.append("\n LogNumber: ");
|
||||
AppendNumberTo(&r, log_number_);
|
||||
}
|
||||
if (has_prev_log_number_) {
|
||||
r.append("\n PrevLogNumber: ");
|
||||
AppendNumberTo(&r, prev_log_number_);
|
||||
}
|
||||
if (has_next_file_number_) {
|
||||
r.append("\n NextFile: ");
|
||||
AppendNumberTo(&r, next_file_number_);
|
||||
}
|
||||
if (has_last_sequence_) {
|
||||
r.append("\n LastSeq: ");
|
||||
AppendNumberTo(&r, last_sequence_);
|
||||
}
|
||||
for (size_t i = 0; i < compact_pointers_.size(); i++) {
|
||||
r.append("\n CompactPointer: ");
|
||||
AppendNumberTo(&r, compact_pointers_[i].first);
|
||||
r.append(" ");
|
||||
r.append(compact_pointers_[i].second.DebugString());
|
||||
}
|
||||
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
|
||||
iter != deleted_files_.end(); ++iter) {
|
||||
r.append("\n DeleteFile: ");
|
||||
AppendNumberTo(&r, iter->first);
|
||||
r.append(" ");
|
||||
AppendNumberTo(&r, iter->second);
|
||||
}
|
||||
for (size_t i = 0; i < new_files_.size(); i++) {
|
||||
const FileMetaData& f = new_files_[i].second;
|
||||
r.append("\n AddFile: ");
|
||||
AppendNumberTo(&r, new_files_[i].first);
|
||||
r.append(" ");
|
||||
AppendNumberTo(&r, f.number);
|
||||
r.append(" ");
|
||||
AppendNumberTo(&r, f.file_size);
|
||||
r.append(" ");
|
||||
r.append(f.smallest.DebugString());
|
||||
r.append(" .. ");
|
||||
r.append(f.largest.DebugString());
|
||||
}
|
||||
r.append("\n}\n");
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
Generated
+106
@@ -0,0 +1,106 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_VERSION_EDIT_H_
|
||||
#define STORAGE_LEVELDB_DB_VERSION_EDIT_H_
|
||||
|
||||
#include <set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class VersionSet;
|
||||
|
||||
struct FileMetaData {
|
||||
FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) {}
|
||||
|
||||
int refs;
|
||||
int allowed_seeks; // Seeks allowed until compaction
|
||||
uint64_t number;
|
||||
uint64_t file_size; // File size in bytes
|
||||
InternalKey smallest; // Smallest internal key served by table
|
||||
InternalKey largest; // Largest internal key served by table
|
||||
};
|
||||
|
||||
class VersionEdit {
|
||||
public:
|
||||
VersionEdit() { Clear(); }
|
||||
~VersionEdit() {}
|
||||
|
||||
void Clear();
|
||||
|
||||
void SetComparatorName(const Slice& name) {
|
||||
has_comparator_ = true;
|
||||
comparator_ = name.ToString();
|
||||
}
|
||||
void SetLogNumber(uint64_t num) {
|
||||
has_log_number_ = true;
|
||||
log_number_ = num;
|
||||
}
|
||||
void SetPrevLogNumber(uint64_t num) {
|
||||
has_prev_log_number_ = true;
|
||||
prev_log_number_ = num;
|
||||
}
|
||||
void SetNextFile(uint64_t num) {
|
||||
has_next_file_number_ = true;
|
||||
next_file_number_ = num;
|
||||
}
|
||||
void SetLastSequence(SequenceNumber seq) {
|
||||
has_last_sequence_ = true;
|
||||
last_sequence_ = seq;
|
||||
}
|
||||
void SetCompactPointer(int level, const InternalKey& key) {
|
||||
compact_pointers_.push_back(std::make_pair(level, key));
|
||||
}
|
||||
|
||||
// Add the specified file at the specified number.
|
||||
// REQUIRES: This version has not been saved (see VersionSet::SaveTo)
|
||||
// REQUIRES: "smallest" and "largest" are smallest and largest keys in file
|
||||
void AddFile(int level, uint64_t file, uint64_t file_size,
|
||||
const InternalKey& smallest, const InternalKey& largest) {
|
||||
FileMetaData f;
|
||||
f.number = file;
|
||||
f.file_size = file_size;
|
||||
f.smallest = smallest;
|
||||
f.largest = largest;
|
||||
new_files_.push_back(std::make_pair(level, f));
|
||||
}
|
||||
|
||||
// Delete the specified "file" from the specified "level".
|
||||
void DeleteFile(int level, uint64_t file) {
|
||||
deleted_files_.insert(std::make_pair(level, file));
|
||||
}
|
||||
|
||||
void EncodeTo(std::string* dst) const;
|
||||
Status DecodeFrom(const Slice& src);
|
||||
|
||||
std::string DebugString() const;
|
||||
|
||||
private:
|
||||
friend class VersionSet;
|
||||
|
||||
typedef std::set<std::pair<int, uint64_t> > DeletedFileSet;
|
||||
|
||||
std::string comparator_;
|
||||
uint64_t log_number_;
|
||||
uint64_t prev_log_number_;
|
||||
uint64_t next_file_number_;
|
||||
SequenceNumber last_sequence_;
|
||||
bool has_comparator_;
|
||||
bool has_log_number_;
|
||||
bool has_prev_log_number_;
|
||||
bool has_next_file_number_;
|
||||
bool has_last_sequence_;
|
||||
|
||||
std::vector<std::pair<int, InternalKey> > compact_pointers_;
|
||||
DeletedFileSet deleted_files_;
|
||||
std::vector<std::pair<int, FileMetaData> > new_files_;
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_
|
||||
Generated
+1585
File diff suppressed because it is too large
Load Diff
Generated
+393
@@ -0,0 +1,393 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// The representation of a DBImpl consists of a set of Versions. The
|
||||
// newest version is called "current". Older versions may be kept
|
||||
// around to provide a consistent view to live iterators.
|
||||
//
|
||||
// Each Version keeps track of a set of Table files per level. The
|
||||
// entire set of versions is maintained in a VersionSet.
|
||||
//
|
||||
// Version,VersionSet are thread-compatible, but require external
|
||||
// synchronization on all accesses.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_VERSION_SET_H_
|
||||
#define STORAGE_LEVELDB_DB_VERSION_SET_H_
|
||||
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "port/port.h"
|
||||
#include "port/thread_annotations.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
namespace log {
|
||||
class Writer;
|
||||
}
|
||||
|
||||
class Compaction;
|
||||
class Iterator;
|
||||
class MemTable;
|
||||
class TableBuilder;
|
||||
class TableCache;
|
||||
class Version;
|
||||
class VersionSet;
|
||||
class WritableFile;
|
||||
|
||||
// Return the smallest index i such that files[i]->largest >= key.
|
||||
// Return files.size() if there is no such file.
|
||||
// REQUIRES: "files" contains a sorted list of non-overlapping files.
|
||||
int FindFile(const InternalKeyComparator& icmp,
|
||||
const std::vector<FileMetaData*>& files, const Slice& key);
|
||||
|
||||
// Returns true iff some file in "files" overlaps the user key range
|
||||
// [*smallest,*largest].
|
||||
// smallest==nullptr represents a key smaller than all keys in the DB.
|
||||
// largest==nullptr represents a key largest than all keys in the DB.
|
||||
// REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges
|
||||
// in sorted order.
|
||||
bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
|
||||
bool disjoint_sorted_files,
|
||||
const std::vector<FileMetaData*>& files,
|
||||
const Slice* smallest_user_key,
|
||||
const Slice* largest_user_key);
|
||||
|
||||
class Version {
|
||||
public:
|
||||
// Lookup the value for key. If found, store it in *val and
|
||||
// return OK. Else return a non-OK status. Fills *stats.
|
||||
// REQUIRES: lock is not held
|
||||
struct GetStats {
|
||||
FileMetaData* seek_file;
|
||||
int seek_file_level;
|
||||
};
|
||||
|
||||
// Append to *iters a sequence of iterators that will
|
||||
// yield the contents of this Version when merged together.
|
||||
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
||||
void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
|
||||
|
||||
Status Get(const ReadOptions&, const LookupKey& key, std::string* val,
|
||||
GetStats* stats);
|
||||
|
||||
// Adds "stats" into the current state. Returns true if a new
|
||||
// compaction may need to be triggered, false otherwise.
|
||||
// REQUIRES: lock is held
|
||||
bool UpdateStats(const GetStats& stats);
|
||||
|
||||
// Record a sample of bytes read at the specified internal key.
|
||||
// Samples are taken approximately once every config::kReadBytesPeriod
|
||||
// bytes. Returns true if a new compaction may need to be triggered.
|
||||
// REQUIRES: lock is held
|
||||
bool RecordReadSample(Slice key);
|
||||
|
||||
// Reference count management (so Versions do not disappear out from
|
||||
// under live iterators)
|
||||
void Ref();
|
||||
void Unref();
|
||||
|
||||
void GetOverlappingInputs(
|
||||
int level,
|
||||
const InternalKey* begin, // nullptr means before all keys
|
||||
const InternalKey* end, // nullptr means after all keys
|
||||
std::vector<FileMetaData*>* inputs);
|
||||
|
||||
// Returns true iff some file in the specified level overlaps
|
||||
// some part of [*smallest_user_key,*largest_user_key].
|
||||
// smallest_user_key==nullptr represents a key smaller than all the DB's keys.
|
||||
// largest_user_key==nullptr represents a key largest than all the DB's keys.
|
||||
bool OverlapInLevel(int level, const Slice* smallest_user_key,
|
||||
const Slice* largest_user_key);
|
||||
|
||||
// Return the level at which we should place a new memtable compaction
|
||||
// result that covers the range [smallest_user_key,largest_user_key].
|
||||
int PickLevelForMemTableOutput(const Slice& smallest_user_key,
|
||||
const Slice& largest_user_key);
|
||||
|
||||
int NumFiles(int level) const { return files_[level].size(); }
|
||||
|
||||
// Return a human readable string that describes this version's contents.
|
||||
std::string DebugString() const;
|
||||
|
||||
private:
|
||||
friend class Compaction;
|
||||
friend class VersionSet;
|
||||
|
||||
class LevelFileNumIterator;
|
||||
|
||||
explicit Version(VersionSet* vset)
|
||||
: vset_(vset),
|
||||
next_(this),
|
||||
prev_(this),
|
||||
refs_(0),
|
||||
file_to_compact_(nullptr),
|
||||
file_to_compact_level_(-1),
|
||||
compaction_score_(-1),
|
||||
compaction_level_(-1) {}
|
||||
|
||||
Version(const Version&) = delete;
|
||||
Version& operator=(const Version&) = delete;
|
||||
|
||||
~Version();
|
||||
|
||||
Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const;
|
||||
|
||||
// Call func(arg, level, f) for every file that overlaps user_key in
|
||||
// order from newest to oldest. If an invocation of func returns
|
||||
// false, makes no more calls.
|
||||
//
|
||||
// REQUIRES: user portion of internal_key == user_key.
|
||||
void ForEachOverlapping(Slice user_key, Slice internal_key, void* arg,
|
||||
bool (*func)(void*, int, FileMetaData*));
|
||||
|
||||
VersionSet* vset_; // VersionSet to which this Version belongs
|
||||
Version* next_; // Next version in linked list
|
||||
Version* prev_; // Previous version in linked list
|
||||
int refs_; // Number of live refs to this version
|
||||
|
||||
// List of files per level
|
||||
std::vector<FileMetaData*> files_[config::kNumLevels];
|
||||
|
||||
// Next file to compact based on seek stats.
|
||||
FileMetaData* file_to_compact_;
|
||||
int file_to_compact_level_;
|
||||
|
||||
// Level that should be compacted next and its compaction score.
|
||||
// Score < 1 means compaction is not strictly needed. These fields
|
||||
// are initialized by Finalize().
|
||||
double compaction_score_;
|
||||
int compaction_level_;
|
||||
};
|
||||
|
||||
class VersionSet {
|
||||
public:
|
||||
VersionSet(const std::string& dbname, const Options* options,
|
||||
TableCache* table_cache, const InternalKeyComparator*);
|
||||
VersionSet(const VersionSet&) = delete;
|
||||
VersionSet& operator=(const VersionSet&) = delete;
|
||||
|
||||
~VersionSet();
|
||||
|
||||
// Apply *edit to the current version to form a new descriptor that
|
||||
// is both saved to persistent state and installed as the new
|
||||
// current version. Will release *mu while actually writing to the file.
|
||||
// REQUIRES: *mu is held on entry.
|
||||
// REQUIRES: no other thread concurrently calls LogAndApply()
|
||||
Status LogAndApply(VersionEdit* edit, port::Mutex* mu)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mu);
|
||||
|
||||
// Recover the last saved descriptor from persistent storage.
|
||||
Status Recover(bool* save_manifest);
|
||||
|
||||
// Return the current version.
|
||||
Version* current() const { return current_; }
|
||||
|
||||
// Return the current manifest file number
|
||||
uint64_t ManifestFileNumber() const { return manifest_file_number_; }
|
||||
|
||||
// Allocate and return a new file number
|
||||
uint64_t NewFileNumber() { return next_file_number_++; }
|
||||
|
||||
// Arrange to reuse "file_number" unless a newer file number has
|
||||
// already been allocated.
|
||||
// REQUIRES: "file_number" was returned by a call to NewFileNumber().
|
||||
void ReuseFileNumber(uint64_t file_number) {
|
||||
if (next_file_number_ == file_number + 1) {
|
||||
next_file_number_ = file_number;
|
||||
}
|
||||
}
|
||||
|
||||
// Return the number of Table files at the specified level.
|
||||
int NumLevelFiles(int level) const;
|
||||
|
||||
// Return the combined file size of all files at the specified level.
|
||||
int64_t NumLevelBytes(int level) const;
|
||||
|
||||
// Return the last sequence number.
|
||||
uint64_t LastSequence() const { return last_sequence_; }
|
||||
|
||||
// Set the last sequence number to s.
|
||||
void SetLastSequence(uint64_t s) {
|
||||
assert(s >= last_sequence_);
|
||||
last_sequence_ = s;
|
||||
}
|
||||
|
||||
// Mark the specified file number as used.
|
||||
void MarkFileNumberUsed(uint64_t number);
|
||||
|
||||
// Return the current log file number.
|
||||
uint64_t LogNumber() const { return log_number_; }
|
||||
|
||||
// Return the log file number for the log file that is currently
|
||||
// being compacted, or zero if there is no such log file.
|
||||
uint64_t PrevLogNumber() const { return prev_log_number_; }
|
||||
|
||||
// Pick level and inputs for a new compaction.
|
||||
// Returns nullptr if there is no compaction to be done.
|
||||
// Otherwise returns a pointer to a heap-allocated object that
|
||||
// describes the compaction. Caller should delete the result.
|
||||
Compaction* PickCompaction();
|
||||
|
||||
// Return a compaction object for compacting the range [begin,end] in
|
||||
// the specified level. Returns nullptr if there is nothing in that
|
||||
// level that overlaps the specified range. Caller should delete
|
||||
// the result.
|
||||
Compaction* CompactRange(int level, const InternalKey* begin,
|
||||
const InternalKey* end);
|
||||
|
||||
// Return the maximum overlapping data (in bytes) at next level for any
|
||||
// file at a level >= 1.
|
||||
int64_t MaxNextLevelOverlappingBytes();
|
||||
|
||||
// Create an iterator that reads over the compaction inputs for "*c".
|
||||
// The caller should delete the iterator when no longer needed.
|
||||
Iterator* MakeInputIterator(Compaction* c);
|
||||
|
||||
// Returns true iff some level needs a compaction.
|
||||
bool NeedsCompaction() const {
|
||||
Version* v = current_;
|
||||
return (v->compaction_score_ >= 1) || (v->file_to_compact_ != nullptr);
|
||||
}
|
||||
|
||||
// Add all files listed in any live version to *live.
|
||||
// May also mutate some internal state.
|
||||
void AddLiveFiles(std::set<uint64_t>* live);
|
||||
|
||||
// Return the approximate offset in the database of the data for
|
||||
// "key" as of version "v".
|
||||
uint64_t ApproximateOffsetOf(Version* v, const InternalKey& key);
|
||||
|
||||
// Return a human-readable short (single-line) summary of the number
|
||||
// of files per level. Uses *scratch as backing store.
|
||||
struct LevelSummaryStorage {
|
||||
char buffer[100];
|
||||
};
|
||||
const char* LevelSummary(LevelSummaryStorage* scratch) const;
|
||||
|
||||
private:
|
||||
class Builder;
|
||||
|
||||
friend class Compaction;
|
||||
friend class Version;
|
||||
|
||||
bool ReuseManifest(const std::string& dscname, const std::string& dscbase);
|
||||
|
||||
void Finalize(Version* v);
|
||||
|
||||
void GetRange(const std::vector<FileMetaData*>& inputs, InternalKey* smallest,
|
||||
InternalKey* largest);
|
||||
|
||||
void GetRange2(const std::vector<FileMetaData*>& inputs1,
|
||||
const std::vector<FileMetaData*>& inputs2,
|
||||
InternalKey* smallest, InternalKey* largest);
|
||||
|
||||
void SetupOtherInputs(Compaction* c);
|
||||
|
||||
// Save current contents to *log
|
||||
Status WriteSnapshot(log::Writer* log);
|
||||
|
||||
void AppendVersion(Version* v);
|
||||
|
||||
Env* const env_;
|
||||
const std::string dbname_;
|
||||
const Options* const options_;
|
||||
TableCache* const table_cache_;
|
||||
const InternalKeyComparator icmp_;
|
||||
uint64_t next_file_number_;
|
||||
uint64_t manifest_file_number_;
|
||||
uint64_t last_sequence_;
|
||||
uint64_t log_number_;
|
||||
uint64_t prev_log_number_; // 0 or backing store for memtable being compacted
|
||||
|
||||
// Opened lazily
|
||||
WritableFile* descriptor_file_;
|
||||
log::Writer* descriptor_log_;
|
||||
Version dummy_versions_; // Head of circular doubly-linked list of versions.
|
||||
Version* current_; // == dummy_versions_.prev_
|
||||
|
||||
// Per-level key at which the next compaction at that level should start.
|
||||
// Either an empty string, or a valid InternalKey.
|
||||
std::string compact_pointer_[config::kNumLevels];
|
||||
};
|
||||
|
||||
// A Compaction encapsulates information about a compaction.
|
||||
class Compaction {
|
||||
public:
|
||||
~Compaction();
|
||||
|
||||
// Return the level that is being compacted. Inputs from "level"
|
||||
// and "level+1" will be merged to produce a set of "level+1" files.
|
||||
int level() const { return level_; }
|
||||
|
||||
// Return the object that holds the edits to the descriptor done
|
||||
// by this compaction.
|
||||
VersionEdit* edit() { return &edit_; }
|
||||
|
||||
// "which" must be either 0 or 1
|
||||
int num_input_files(int which) const { return inputs_[which].size(); }
|
||||
|
||||
// Return the ith input file at "level()+which" ("which" must be 0 or 1).
|
||||
FileMetaData* input(int which, int i) const { return inputs_[which][i]; }
|
||||
|
||||
// Maximum size of files to build during this compaction.
|
||||
uint64_t MaxOutputFileSize() const { return max_output_file_size_; }
|
||||
|
||||
// Is this a trivial compaction that can be implemented by just
|
||||
// moving a single input file to the next level (no merging or splitting)
|
||||
bool IsTrivialMove() const;
|
||||
|
||||
// Add all inputs to this compaction as delete operations to *edit.
|
||||
void AddInputDeletions(VersionEdit* edit);
|
||||
|
||||
// Returns true if the information we have available guarantees that
|
||||
// the compaction is producing data in "level+1" for which no data exists
|
||||
// in levels greater than "level+1".
|
||||
bool IsBaseLevelForKey(const Slice& user_key);
|
||||
|
||||
// Returns true iff we should stop building the current output
|
||||
// before processing "internal_key".
|
||||
bool ShouldStopBefore(const Slice& internal_key);
|
||||
|
||||
// Release the input version for the compaction, once the compaction
|
||||
// is successful.
|
||||
void ReleaseInputs();
|
||||
|
||||
private:
|
||||
friend class Version;
|
||||
friend class VersionSet;
|
||||
|
||||
Compaction(const Options* options, int level);
|
||||
|
||||
int level_;
|
||||
uint64_t max_output_file_size_;
|
||||
Version* input_version_;
|
||||
VersionEdit edit_;
|
||||
|
||||
// Each compaction reads inputs from "level_" and "level_+1"
|
||||
std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
|
||||
|
||||
// State used to check for number of overlapping grandparent files
|
||||
// (parent == level_ + 1, grandparent == level_ + 2)
|
||||
std::vector<FileMetaData*> grandparents_;
|
||||
size_t grandparent_index_; // Index in grandparent_starts_
|
||||
bool seen_key_; // Some output key has been seen
|
||||
int64_t overlapped_bytes_; // Bytes of overlap between current output
|
||||
// and grandparent files
|
||||
|
||||
// State for implementing IsBaseLevelForKey
|
||||
|
||||
// level_ptrs_ holds indices into input_version_->levels_: our state
|
||||
// is that we are positioned at one of the file ranges for each
|
||||
// higher level than the ones involved in this compaction (i.e. for
|
||||
// all L >= level_ + 2).
|
||||
size_t level_ptrs_[config::kNumLevels];
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_VERSION_SET_H_
|
||||
Generated
+150
@@ -0,0 +1,150 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// WriteBatch::rep_ :=
|
||||
// sequence: fixed64
|
||||
// count: fixed32
|
||||
// data: record[count]
|
||||
// record :=
|
||||
// kTypeValue varstring varstring |
|
||||
// kTypeDeletion varstring
|
||||
// varstring :=
|
||||
// len: varint32
|
||||
// data: uint8[len]
|
||||
|
||||
#include "leveldb/write_batch.h"
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/memtable.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
|
||||
static const size_t kHeader = 12;
|
||||
|
||||
WriteBatch::WriteBatch() { Clear(); }
|
||||
|
||||
WriteBatch::~WriteBatch() {}
|
||||
|
||||
WriteBatch::Handler::~Handler() {}
|
||||
|
||||
void WriteBatch::Clear() {
|
||||
rep_.clear();
|
||||
rep_.resize(kHeader);
|
||||
}
|
||||
|
||||
size_t WriteBatch::ApproximateSize() const { return rep_.size(); }
|
||||
|
||||
Status WriteBatch::Iterate(Handler* handler) const {
|
||||
Slice input(rep_);
|
||||
if (input.size() < kHeader) {
|
||||
return Status::Corruption("malformed WriteBatch (too small)");
|
||||
}
|
||||
|
||||
input.remove_prefix(kHeader);
|
||||
Slice key, value;
|
||||
int found = 0;
|
||||
while (!input.empty()) {
|
||||
found++;
|
||||
char tag = input[0];
|
||||
input.remove_prefix(1);
|
||||
switch (tag) {
|
||||
case kTypeValue:
|
||||
if (GetLengthPrefixedSlice(&input, &key) &&
|
||||
GetLengthPrefixedSlice(&input, &value)) {
|
||||
handler->Put(key, value);
|
||||
} else {
|
||||
return Status::Corruption("bad WriteBatch Put");
|
||||
}
|
||||
break;
|
||||
case kTypeDeletion:
|
||||
if (GetLengthPrefixedSlice(&input, &key)) {
|
||||
handler->Delete(key);
|
||||
} else {
|
||||
return Status::Corruption("bad WriteBatch Delete");
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return Status::Corruption("unknown WriteBatch tag");
|
||||
}
|
||||
}
|
||||
if (found != WriteBatchInternal::Count(this)) {
|
||||
return Status::Corruption("WriteBatch has wrong count");
|
||||
} else {
|
||||
return Status::OK();
|
||||
}
|
||||
}
|
||||
|
||||
int WriteBatchInternal::Count(const WriteBatch* b) {
|
||||
return DecodeFixed32(b->rep_.data() + 8);
|
||||
}
|
||||
|
||||
void WriteBatchInternal::SetCount(WriteBatch* b, int n) {
|
||||
EncodeFixed32(&b->rep_[8], n);
|
||||
}
|
||||
|
||||
SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) {
|
||||
return SequenceNumber(DecodeFixed64(b->rep_.data()));
|
||||
}
|
||||
|
||||
void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) {
|
||||
EncodeFixed64(&b->rep_[0], seq);
|
||||
}
|
||||
|
||||
void WriteBatch::Put(const Slice& key, const Slice& value) {
|
||||
WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
|
||||
rep_.push_back(static_cast<char>(kTypeValue));
|
||||
PutLengthPrefixedSlice(&rep_, key);
|
||||
PutLengthPrefixedSlice(&rep_, value);
|
||||
}
|
||||
|
||||
void WriteBatch::Delete(const Slice& key) {
|
||||
WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
|
||||
rep_.push_back(static_cast<char>(kTypeDeletion));
|
||||
PutLengthPrefixedSlice(&rep_, key);
|
||||
}
|
||||
|
||||
void WriteBatch::Append(const WriteBatch& source) {
|
||||
WriteBatchInternal::Append(this, &source);
|
||||
}
|
||||
|
||||
namespace {
|
||||
class MemTableInserter : public WriteBatch::Handler {
|
||||
public:
|
||||
SequenceNumber sequence_;
|
||||
MemTable* mem_;
|
||||
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
mem_->Add(sequence_, kTypeValue, key, value);
|
||||
sequence_++;
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
mem_->Add(sequence_, kTypeDeletion, key, Slice());
|
||||
sequence_++;
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* memtable) {
|
||||
MemTableInserter inserter;
|
||||
inserter.sequence_ = WriteBatchInternal::Sequence(b);
|
||||
inserter.mem_ = memtable;
|
||||
return b->Iterate(&inserter);
|
||||
}
|
||||
|
||||
void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
|
||||
assert(contents.size() >= kHeader);
|
||||
b->rep_.assign(contents.data(), contents.size());
|
||||
}
|
||||
|
||||
void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) {
|
||||
SetCount(dst, Count(dst) + Count(src));
|
||||
assert(src->rep_.size() >= kHeader);
|
||||
dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader);
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
|
||||
#define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class MemTable;
|
||||
|
||||
// WriteBatchInternal provides static methods for manipulating a
|
||||
// WriteBatch that we don't want in the public WriteBatch interface.
|
||||
class WriteBatchInternal {
|
||||
public:
|
||||
// Return the number of entries in the batch.
|
||||
static int Count(const WriteBatch* batch);
|
||||
|
||||
// Set the count for the number of entries in the batch.
|
||||
static void SetCount(WriteBatch* batch, int n);
|
||||
|
||||
// Return the sequence number for the start of this batch.
|
||||
static SequenceNumber Sequence(const WriteBatch* batch);
|
||||
|
||||
// Store the specified number as the sequence number for the start of
|
||||
// this batch.
|
||||
static void SetSequence(WriteBatch* batch, SequenceNumber seq);
|
||||
|
||||
static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); }
|
||||
|
||||
static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); }
|
||||
|
||||
static void SetContents(WriteBatch* batch, const Slice& contents);
|
||||
|
||||
static Status InsertInto(const WriteBatch* batch, MemTable* memtable);
|
||||
|
||||
static void Append(WriteBatch* dst, const WriteBatch* src);
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
|
||||
Reference in New Issue
Block a user