From 51d8b6fdfabd132016dd7f7558fe31fe3c999991 Mon Sep 17 00:00:00 2001 From: fengyang Date: Sat, 1 Apr 2023 10:11:00 +0800 Subject: [PATCH 1/2] =?UTF-8?q?table=20build=20=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/builder.rs | 24 ++++++++++++------------ src/db/builder_test.rs | 10 +++++++++- src/db/skip_list.rs | 4 ---- src/db/table_cache.rs | 5 +++++ src/traits/iterator.rs | 3 --- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/db/builder.rs b/src/db/builder.rs index d6d9491..3814e10 100644 --- a/src/db/builder.rs +++ b/src/db/builder.rs @@ -64,7 +64,7 @@ impl BuildTable { meta.get_smallest().decode_from(&iter.key()); // todo 逻辑 check // 调用迭代器，依次将每个键-值对加入 TableBuilder - while iter.valid() && iter.has_next(){ + while iter.valid() { iter.next(); let key = iter.key(); @@ -80,17 +80,17 @@ impl BuildTable { assert!(meta.get_file_size() > 0); } - // // Finish and check for file errors - // // 将文件刷新到磁盘 - // if s.is_ok() { - // let rs:io::Result<()> = writableFile.sync_data(); - // if rs.is_ok() { - // s = Status::default(); - // }else{ - // s = Status::wrapper_str(LevelError::KIOError, rs.unwrap_err().to_string().as_str()); - // } - // } - // // 关闭文件 + // Finish and check for file errors + // 将文件刷新到磁盘 + if s.is_ok() { + let rs:io::Result<()> = writableFile.sync_data(); + if rs.is_ok() { + s = Status::default(); + }else{ + s = Status::wrapper_str(LevelError::KIOError, rs.unwrap_err().to_string().as_str()); + } + } + // 关闭文件 // if s.is_ok() { // writableFile.close // } diff --git a/src/db/builder_test.rs b/src/db/builder_test.rs index 44c68c4..7f1746a 100644 --- a/src/db/builder_test.rs +++ b/src/db/builder_test.rs @@ -1,8 +1,16 @@ mod test { + use crate::db::builder::BuildTable; + use crate::db::table_cache::TableCache; + use crate::util::options::Options; + use crate::util::slice::Slice; + #[test] - fn test_() { + fn test_build_table() { + // BuildTable::build_table(&Slice::from("a"), Options::default(), + // TableCache::new(), + // Box::new()); println!("get_name: {}", "a"); } diff --git a/src/db/skip_list.rs b/src/db/skip_list.rs index 34679ee..4d13c12 100644 --- a/src/db/skip_list.rs +++ b/src/db/skip_list.rs @@ -470,10 +470,6 @@ impl DataIterator for Iter { } } - fn has_next(&self) -> bool { - todo!() - } - fn pre(&mut self) { todo!() } diff --git a/src/db/table_cache.rs b/src/db/table_cache.rs index ad6079b..458eb13 100644 --- a/src/db/table_cache.rs +++ b/src/db/table_cache.rs @@ -11,6 +11,11 @@ struct Table {} pub struct TableCache {} impl TableCache { + pub fn new() -> Self { + Self { + + } + } /// 从缓存中获取Table /// /// # Arguments diff --git a/src/traits/iterator.rs b/src/traits/iterator.rs index 41a331a..2ffaad4 100644 --- a/src/traits/iterator.rs +++ b/src/traits/iterator.rs @@ -71,9 +71,6 @@ pub trait DataIterator { /// ``` fn next(&mut self); - /// 是否存在下一个元素 - fn has_next(&self) -> bool; - /// 定位到上一个元素 /// /// # Arguments -- Gitee From c309e0077165fa03f959cff9a7a87f2be72385ed Mon Sep 17 00:00:00 2001 From: fengyang Date: Mon, 3 Apr 2023 20:21:56 +0800 Subject: [PATCH 2/2] =?UTF-8?q?BlockBuilder=20=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 +++-- src/db/builder.rs | 54 ++++++++++++++++++++------------- src/db/skip_list.rs | 4 +++ src/db/table_cache.rs | 12 ++++---- src/table/block_builder.rs | 32 +++++++++++++++++++- src/table/filter_block.rs | 38 ++++++++++++----------- src/table/filter_block_test.rs | 6 ++-- src/table/mod.rs | 4 ++- src/table/table.rs | 25 ++++++++++++++++ src/table/table_builder.rs | 55 +++++++++++++++++++++++++++------- src/table/table_test.rs | 0 src/traits/iterator.rs | 3 ++ 12 files changed, 177 insertions(+), 64 deletions(-) create mode 100644 src/table/table.rs create mode 100644 src/table/table_test.rs diff --git a/README.md b/README.md index 6f6197b..f05c752 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,11 @@ RUSTFLAGS='--cfg CORE_DEBUG="false"' cargo build --release | table.Iterator(Block::Iter) | fengyang | 0% | | IteratorWrapper | kazeseiriou | 0% | | db.MemTable(MemTable, MemTableIterator) | wangboo,tzcyujunyong | 20% | -| db.Builder | fengyang | 20% | +| db.Builder | fengyang | 85% | +| table.table_builder | fengyang | 30% | | table.Block | fengyang | 30% | -| table.BlockBuilder, table.FilterBlockBuilder | | | +| table.BlockBuilder | fengyang | 30% | +| table.FilterBlockBuilder | fengyang | 30% | | FilterBlock, FilterBlockReader | fengyang | 80% | | SSTable | fengyang | 0% | | table.Table | peach,tzcyujunyong | | @@ -102,7 +104,7 @@ RUSTFLAGS='--cfg CORE_DEBUG="false"' cargo build --release | db.VersionEdit(Tag, VersionEdit, FileMetaData) | fengyang | 20% | | db.VersionSet(Version, LevelFileNumIterator, SaverState) | fengyang | 20% | | WriteBatch | tzcyujunyong,wangboo | 50% | -| table.table_builder | | 30% | +| table.table | | 30% | | db.filename | | | | | 半支烟 | 40% | diff --git a/src/db/builder.rs b/src/db/builder.rs index 3814e10..9140e5e 100644 --- a/src/db/builder.rs +++ b/src/db/builder.rs @@ -6,10 +6,11 @@ use std::sync::Arc; use crate::db::file_meta_data::FileMetaData; use crate::db::filename::FileName; use crate::db::table_cache::TableCache; +use crate::table::table::Table; use crate::table::table_builder::TableBuilder; use crate::traits::DataIterator; use crate::util::env::Env; -use crate::util::options::{Options}; +use crate::util::options::{Options, ReadOptions}; use crate::util::Result; use crate::util::slice::Slice; use crate::util::status::{LevelError, Status}; @@ -51,6 +52,8 @@ impl BuildTable { // 生成一个 SSTable 文件名 let file_name = FileName::table_file_name(dbname, meta.get_number()); + let mut s : Status = Status::default(); + if iter.valid() { let fileRS: Result = env.new_writable_file(&file_name); if(!fileRS.is_ok()){ @@ -62,7 +65,7 @@ impl BuildTable { let builder: TableBuilder = TableBuilder::new_with_writable_file(options, writableFile); meta.get_smallest().decode_from(&iter.key()); - // todo 逻辑 check + // 调用迭代器，依次将每个键-值对加入 TableBuilder while iter.valid() { iter.next(); @@ -74,7 +77,7 @@ impl BuildTable { // Finish and check for builder errors // 调用 TableBuilder 的 Finish 函数生成 SSTable 文件 - let mut s : Status = builder.finish(); + s = builder.finish(); if s.is_ok() { meta.set_file_size(builder.get_file_size()); assert!(meta.get_file_size() > 0); @@ -96,26 +99,35 @@ impl BuildTable { // } if s.is_ok() { -// // Verify that the table is usable -// Iterator* it = table_cache->NewIterator(ReadOptions(), -// meta->number, -// meta->file_size); -// s = it->status(); -// delete it; + let readOptions = ReadOptions::default(); + // Verify that the table is usable + let it: Box = table_cache.new_iterator(&readOptions, + meta.get_number(), + meta.get_file_size() as usize, + &Table::new()) + .expect("table_cache.new_iterator error"); + s = it.status(); } } // if end -// // Check for input iterator errors -// if (!iter->status().ok()) { -// s = iter->status(); -// } -// -// if (s.ok() && meta->file_size > 0) { -// // Keep it -// } else { -// env->DeleteFile(fname); -// } -// return s; - Err(Status::wrapper_str(LevelError::KBadRecord, "a")) + // Check for input iterator errors + if !iter.status().is_ok() { + s = iter.status(); + } + + if s.is_ok() && meta.get_file_size() > 0 { + // Keep it + } else { + // DeleteFile fname + // todo + } + + if s.is_ok() { + // todo + // return Ok(meta); + return Ok(FileMetaData::default()); + }else{ + return Err(s); + } } } \ No newline at end of file diff --git a/src/db/skip_list.rs b/src/db/skip_list.rs index 4d13c12..a1643db 100644 --- a/src/db/skip_list.rs +++ b/src/db/skip_list.rs @@ -484,4 +484,8 @@ impl DataIterator for Iter { fn value(&self) -> UnsafeSlice { todo!() } + + fn status(&self) -> Status { + todo!() + } } \ No newline at end of file diff --git a/src/db/table_cache.rs b/src/db/table_cache.rs index 458eb13..5824617 100644 --- a/src/db/table_cache.rs +++ b/src/db/table_cache.rs @@ -1,13 +1,11 @@ +use crate::table::table::Table; use crate::traits::DataIterator; +use crate::util::options::ReadOptions; use crate::util::slice::Slice; use crate::util::Result; struct Saver {} -struct ReadOptions {} - -struct Table {} - pub struct TableCache {} impl TableCache { @@ -33,7 +31,7 @@ impl TableCache { /// ``` /// /// ``` - fn get(&self, _options: &ReadOptions, _file_number: u64, _file_size: usize, _k: &Slice, _arg: &mut Saver, _handle_result: F) + pub fn get(&self, _options: &ReadOptions, _file_number: u64, _file_size: usize, _k: &Slice, _arg: &mut Saver, _handle_result: F) where F: FnMut(&mut Saver, &Slice, &Slice) -> Result<()> { () } @@ -50,7 +48,7 @@ impl TableCache { /// ``` /// /// ``` - fn evict(&mut self, _file_number: u64) { + pub fn evict(&mut self, _file_number: u64) { todo!() } @@ -70,7 +68,7 @@ impl TableCache { /// ``` /// /// ``` - fn new_iterator(&self, _options: &ReadOptions, _file_number: u64, _file_size: usize, _table: &Table) -> Result> { + pub fn new_iterator(&self, _options: &ReadOptions, _file_number: u64, _file_size: usize, _table: &Table) -> Result> { todo!() } } \ No newline at end of file diff --git a/src/table/block_builder.rs b/src/table/block_builder.rs index 2d6467d..3ca4eec 100644 --- a/src/table/block_builder.rs +++ b/src/table/block_builder.rs @@ -1,9 +1,34 @@ +use std::fs::File; +use std::sync::Arc; use crate::util::options::Options; use crate::util::slice::Slice; use crate::util::Result; +use crate::util::status::Status; -pub struct BlockBuilder {} +// 智能指针 Rc, 引用计数器，用来记录一个值是否被使用，如果计数为零可清除。 +// 适用于堆中数据需要被程序多部分使用，但编译时不能确定谁最后完成。 + +// Arc 是一种能够使得数据在线程间安全共享的智能指针. +// Arc会追踪这个指针的所有拷贝，当最后一份拷贝离开作用域时，它就会安全释放内存。 + +// 智能指针 Box。 box 允许你将一个值放在堆上而不是栈上。留在栈上的则是指向堆数据的指针。 +pub struct BlockBuilder { + // 在 BlockBuilder 初始化时，指定的配置项 + options: Box, + index_block_options: Box, + + // SSTable 生成后的文件 + file: Arc, + + offset: u64, + status: Status, + + // 生成 SSTable 中的数据区域 + data_block: Arc, + // 生成 SSTable 中的数据索引区域 + index_block: Arc, +} impl BlockBuilder { pub fn new(options: &Options) -> Self { @@ -27,6 +52,7 @@ impl BlockBuilder { pub fn add(&mut self, _key: &Slice, _value: &Slice) { todo!() } + /// 重置builder /// /// # Examples @@ -37,6 +63,7 @@ impl BlockBuilder { pub fn reset(&mut self) { todo!() } + /// 构造block /// /// @@ -48,6 +75,7 @@ impl BlockBuilder { pub fn finish(&mut self) -> Result { todo!() } + /// 判断builder是否为空 /// /// # Examples @@ -58,6 +86,7 @@ impl BlockBuilder { pub fn empty(&self) -> bool { todo!() } + /// 估算当前的block大小 /// /// # Examples @@ -68,4 +97,5 @@ impl BlockBuilder { pub fn current_size_estimate(&self) -> usize { todo!() } + } \ No newline at end of file diff --git a/src/table/filter_block.rs b/src/table/filter_block.rs index 183c678..625f2c9 100644 --- a/src/table/filter_block.rs +++ b/src/table/filter_block.rs @@ -14,7 +14,9 @@ const FILTER_BASE: usize = 1 << FILTER_BASE_LG; /// /// meta block 构建器 /// -pub trait FilterBlock { +pub trait FilterBlock { + #[inline] + fn new_with_policy(policy: Box) -> Self; /// /// 构造一个 FilterBlockBuilder @@ -29,14 +31,10 @@ pub trait FilterBlock { /// # Examples /// /// ``` - /// use std::sync::Arc; - /// use level_db_rust::util::filter_policy::BloomFilterPolicy; /// - /// let policy = Arc::new(BloomFilterPolicy::new(2)); - /// let filter_block: FilterBlockBuilder = FilterBlockBuilder::new_with_policy(policy); /// ``` #[inline] - fn new_with_policy(policy: Arc, capacity: usize) -> Self; + fn new_with_policy_capacity(policy: Box, capacity: usize) -> Self; /// 设置block的起始位置 /// @@ -80,7 +78,7 @@ pub trait FilterBlock { /// ``` fn finish(&mut self) -> Result; - fn get_policy(&self) -> Box<&FP>; + fn get_policy(&self) -> Box<&dyn FilterPolicy>; fn get_keys(&self) -> Vec; @@ -94,8 +92,8 @@ pub trait FilterBlock { } /// SSTable 文件里面的 meta block 构建器, 按内存里面指定的格式整理在内存中 -pub struct FilterBlockBuilder { - policy: Arc, +pub struct FilterBlockBuilder { + policy: Box, // Flattened key contents keys: Vec, // Starting index in keys_ of each key @@ -107,8 +105,8 @@ pub struct FilterBlockBuilder { filter_offsets: Vec, } -pub struct FilterBlockReader { - policy: Arc, +pub struct FilterBlockReader { + policy: Box, // Pointer to filter data (at block-start) data: Vec, // Pointer to beginning of offset array (at block-end) @@ -119,8 +117,12 @@ pub struct FilterBlockReader { base_lg: usize } -impl FilterBlock for FilterBlockBuilder { - fn new_with_policy(policy: Arc, capacity: usize) -> Self { +impl FilterBlock for FilterBlockBuilder { + fn new_with_policy(policy: Box) -> Self { + FilterBlock::new_with_policy_capacity(policy, 64) + } + + fn new_with_policy_capacity(policy: Box, capacity: usize) -> Self { let keys:Vec = Vec::with_capacity(capacity); let start:Vec = Vec::with_capacity(capacity); let result:Vec = Vec::with_capacity(capacity); @@ -185,7 +187,7 @@ impl FilterBlock for FilterBlockBuilder { Ok(Slice::from_buf(&self.result)) } - fn get_policy(&self) -> Box<&FP> { + fn get_policy(&self) -> Box<&dyn FilterPolicy> { Box::new(self.policy.as_ref()) } @@ -210,7 +212,7 @@ impl FilterBlock for FilterBlockBuilder { } } -impl FilterBlockBuilder { +impl FilterBlockBuilder { fn generate_filter(&mut self) { let num_keys = self.start.len(); @@ -254,8 +256,8 @@ impl FilterBlockBuilder { } } -impl FilterBlockReader { - pub fn new_with_policy(policy: Arc, contents: Slice) -> Self { +impl FilterBlockReader { + pub fn new_with_policy(policy: Box, contents: Slice) -> Self { let data = Vec::new(); let offset = Vec::new(); @@ -290,7 +292,7 @@ impl FilterBlockReader { todo!() } - pub fn get_policy(&self) -> Box<&FP> { + pub fn get_policy(&self) -> Box<&dyn FilterPolicy> { Box::new(self.policy.as_ref()) } diff --git a/src/table/filter_block_test.rs b/src/table/filter_block_test.rs index 0e57a1a..9582b2a 100644 --- a/src/table/filter_block_test.rs +++ b/src/table/filter_block_test.rs @@ -113,7 +113,7 @@ mod test { #[test] fn test_filter_block_new_with_policy() { - let policy = Arc::new(TestHashFilter::new()); + let policy = Box::new(TestHashFilter::new()); let filter_block: FilterBlockBuilder = FilterBlockBuilder::new_with_policy(policy, 10); @@ -129,7 +129,7 @@ mod test { #[test] fn test_filter_block_reader_new_with_policy_empty_content() { - let policy = Arc::new(TestHashFilter::new()); + let policy = Box::new(TestHashFilter::new()); let contents = Slice::default(); let filter_block_reader: FilterBlockReader = FilterBlockReader::new_with_policy(policy, contents); @@ -145,7 +145,7 @@ mod test { #[test] fn test_filter_block_new_with_policy_and_addkey() { - let policy = Arc::new(TestHashFilter::new()); + let policy = Box::new(TestHashFilter::new()); let mut filter_block_builder: FilterBlockBuilder = FilterBlockBuilder::new_with_policy( policy, 10); diff --git a/src/table/mod.rs b/src/table/mod.rs index cc22b85..f4e0a94 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -8,4 +8,6 @@ pub mod ss_table; mod ss_table_test; pub mod iterator_wrapper; pub mod table_builder; -mod table_builder_test; \ No newline at end of file +mod table_builder_test; +pub mod table; +mod table_test; \ No newline at end of file diff --git a/src/table/table.rs b/src/table/table.rs new file mode 100644 index 0000000..65db038 --- /dev/null +++ b/src/table/table.rs @@ -0,0 +1,25 @@ +use std::fs::File; +use std::sync::Arc; +use crate::util::options::Options; +use crate::util::Result; +use crate::util::status::Status; + +pub struct Table { + rep: Rep +} + +struct Rep { + options: Box, + status: Status, + file: Arc, +} + +impl Table { + pub fn new() -> Self{ + todo!() + } + + pub fn open(&self, options:&Options, randomAccessFile:&File, file_size: u64, table:&Table) -> Result{ + todo!() + } +} \ No newline at end of file diff --git a/src/table/table_builder.rs b/src/table/table_builder.rs index 1c89579..4615fdc 100644 --- a/src/table/table_builder.rs +++ b/src/table/table_builder.rs @@ -2,7 +2,7 @@ use std::borrow::Borrow; use std::fs::File; use std::sync::Arc; use crate::table::block_builder::BlockBuilder; -use crate::table::filter_block::FilterBlockBuilder; +use crate::table::filter_block::{FilterBlock, FilterBlockBuilder}; use crate::table::format::BlockHandle; use crate::traits::filter_policy_trait::FilterPolicy; use crate::util::options::{CompressionType, Options}; @@ -11,21 +11,39 @@ use crate::util::status::Status; use crate::util::unsafe_slice::UnsafeSlice; pub struct TableBuilder { - rep: Rep + rep: Box } -struct Rep { - // options: Box, - // index_block_options: Options, +/// TableBuilder Rep 结构体， 内部使用 +struct Rep<> { + options: Box, + index_block_options: Box, + + // SSTable 生成后的文件 file: Arc, + offset: u64, status: Status, - // data_block: BlockBuilder, - // index_block: BlockBuilder, + + // 生成 SSTable 中的数据区域 + data_block: BlockBuilder, + // 生成 SSTable 中的数据索引区域 + index_block: BlockBuilder, + last_key: Slice, num_entries: u64, // Either Finish() or Abandon() has been called. closed: bool, + + // 生成 SSTable 中的元数据区域 + filter_block: Option, + // 判断是否需要生成 SSTable中的数据索引， SSTable中每次生成一个完整的块之后，需要将该值置为 true， 说明需要为该块添加索引 + pending_index_entry: bool, + // Handle to add to index block + // pending_handle 记录需要生成数据索引的数据块在 SSTable 中的偏移量和大小 + pending_handle: BlockHandle, + + compressed_output: Slice, } impl TableBuilder { @@ -78,16 +96,33 @@ impl TableBuilder { impl Rep { pub fn new(opt: &Options, writableFile: Arc) -> Self { + // todo 如何赋值？ Box::new(opt) + let options = Box::new(Default::default()); + let index_block_options = Box::new(Default::default()); + + let mut filter_block: Option; + if opt.filter_policy.is_none() { + filter_block = None; + }else { + filter_block = Some(FilterBlockBuilder::new_with_policy(opt.filter_policy.unwrap())); + } + Self { - // options: Box::new(*opt), + options, + index_block_options, file: writableFile, offset: 0, - // todo default Status::OK + // default Status::OK status: Status::default(), - // data_block: BlockBuilder::new(&opt), + data_block: BlockBuilder::new(&options.as_ref()), + index_block: BlockBuilder::new(&index_block_options.as_ref()), last_key: Default::default(), num_entries: 0, closed: false, + filter_block, + pending_index_entry: false, + pending_handle: Default::default(), + compressed_output: Default::default(), } } } \ No newline at end of file diff --git a/src/table/table_test.rs b/src/table/table_test.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/traits/iterator.rs b/src/traits/iterator.rs index 2ffaad4..69bb7c1 100644 --- a/src/traits/iterator.rs +++ b/src/traits/iterator.rs @@ -1,4 +1,5 @@ use crate::util::slice::Slice; +use crate::util::status::Status; use crate::util::unsafe_slice::UnsafeSlice; /// @@ -114,4 +115,6 @@ pub trait DataIterator { /// ``` fn value(&self) -> UnsafeSlice; + fn status(&self) -> Status; + } -- Gitee