diff --git a/src/db/db_format.rs b/src/db/db_format.rs index 4b4a7c791d252768c90e29f2a4bda4591af1de86..59121819255092631e779fbda2b744ec8b31f938 100644 --- a/src/db/db_format.rs +++ b/src/db/db_format.rs @@ -1,10 +1,12 @@ use std::cmp::Ordering; use std::io::Write; +use std::ops::Deref; use std::sync::Arc; use crate::db::db_format::ValueType::{KTypeDeletion, KTypeValue}; use crate::db::file_meta_data::FileMetaData; use crate::traits::comparator_trait::Comparator; -use crate::util::coding::{Encoder, varint_length}; +use crate::util::coding::{Decoder, Encoder, varint_length}; +use crate::util::comparator::BytewiseComparatorImpl; use crate::util::slice::Slice; use crate::util::unsafe_slice::UnsafeSlice; @@ -123,8 +125,17 @@ impl ParsedInternalKey { } /// Returns the user key portion of an internal key. - pub fn extract_user_key(internal_key: Slice) -> Slice { - todo!() + pub unsafe fn extract_user_key(internal_key: Slice) -> Slice { + ParsedInternalKey::extract_user_key_with_u8(internal_key.deref()) + } + + /// Returns the user key portion of an internal key. + pub unsafe fn extract_user_key_with_u8(internal_key: &[u8]) -> Slice { + // assert!(internal_key.len() >= 8); + + Slice::from_buf(internal_key) + // todo + // Slice::from_buf(internal_key.deref(), internal_key.size() - 8) } } @@ -193,7 +204,7 @@ impl InternalKey { } pub fn user_key(self) -> Slice { - ParsedInternalKey::extract_user_key(self.rep_) + unsafe { ParsedInternalKey::extract_user_key(self.rep_) } } pub fn set_from(self, p: ParsedInternalKey) { @@ -208,13 +219,17 @@ impl InternalKey { impl Default for InternalKeyComparator { fn default() -> Self { - todo!() + Self { + user_comparator_: Arc::new(InternalKeyComparator::create()), + } } } impl InternalKeyComparator { - pub fn create(_cmp: Box) -> Box { - todo!() + pub fn create() -> InternalKeyComparator { + Self { + user_comparator_: Arc::new(BytewiseComparatorImpl::default()) + } } pub fn user_comparator(&self) -> Box { @@ -230,12 +245,28 @@ impl InternalKeyComparator { /// InternalKeyComparator 比较器: 用来比较内部键（Internal Key）。 /// 内部键值是为了方便处理，将原普通键、序列号和值类型组成的新键。 impl Comparator for InternalKeyComparator { - // fn new(c: Box) -> InternalKeyComparator { - // todo!() - // } + fn compare(&self, akey: &[u8], bkey: &[u8]) -> Option { + // Order by: + // increasing user key (according to user-supplied comparator) + // decreasing sequence number + // decreasing type (though sequence# should be enough to disambiguate) + let mut r = self.user_comparator_.compare( + unsafe { ParsedInternalKey::extract_user_key_with_u8(akey) }.deref(), + unsafe { ParsedInternalKey::extract_user_key_with_u8(bkey) }.deref() + ); + + if r.unwrap() == Ordering::Equal { + let anum = Decoder::with_buf(akey).get_fixed64().unwrap(); + let bnum = Decoder::with_buf(bkey).get_fixed64().unwrap(); + + if anum > bnum { + r = Option::from(Ordering::Less); + } else if anum < bnum { + r = Option::from(Ordering::Greater); + } + } - fn compare(&self, _a: &[u8], _b: &[u8]) -> Option { - todo!() + r } fn get_name(&self) -> String { diff --git a/src/table/block_builder.rs b/src/table/block_builder.rs index 97f20e3bf6a9f2ad9fb4c07d0b6b80b3a49e2f69..aec7fe4f8862b987035d129c663226bb6da4980e 100644 --- a/src/table/block_builder.rs +++ b/src/table/block_builder.rs @@ -1,4 +1,6 @@ +use std::cmp::{min, Ordering}; use std::fs::File; +use std::ops::Deref; use std::sync::Arc; use crate::util::options::{Options, OptionsPtr}; use crate::util::slice::Slice; @@ -23,23 +25,39 @@ pub struct BlockBuilder { options: OptionsPtr, // 目标缓冲区，也就是按照输出格式处理好的内存区域 - buffer: Slice, + buffer: Vec, - // SSTable 生成后的文件 - file: Arc, + // Restart points + restarts: Vec, - offset: u64, - status: Status, + // Number of entries emitted since restart + counter: u32, + // Has Finish() been called? + finished: bool, - // 生成 SSTable 中的数据区域 - data_block: BlockBuilderPtr, - // 生成 SSTable 中的数据索引区域 - index_block: BlockBuilderPtr, + last_key: String } impl BlockBuilder { pub fn new(options: OptionsPtr) -> Self { - todo!() + assert!(options.block_restart_interval >= 1); + + let mut restarts = vec![]; + // First restart point is at offset 0 + restarts.push(0); + + Self { + options, + buffer: vec![], + restarts, + counter: 0, + finished: false, + last_key: "".to_string(), + } + } + + pub fn get_restarts(self) -> Vec { + self.restarts } /// 向datablock增加entry @@ -56,8 +74,29 @@ impl BlockBuilder { /// ``` /// /// ``` - pub fn add(&mut self, _key: &Slice, _value: &Slice) { - todo!() + pub fn add(&mut self, key: Slice, value: Slice) { + let last_key_piece = Slice::from(&self.last_key); + assert!(!self.finished); + assert!(self.counter <= self.options.block_restart_interval); + assert!(!self.buffer.is_empty() // No values yet? + // > 0 + || self.options.cmp.compare(key.deref(), last_key_piece.deref()).unwrap() != Ordering::Less + ); + + let mut shared = 0; + + if self.counter < self.options.block_restart_interval { + // See how much sharing to do with previous string + let min_length = min(last_key_piece.size(), key.len()); + while ((shared < min_length) && (last_key_piece[shared] == key[shared])) { + shared += 1; + } + }else { + // Restart compression + self.restarts.push(self.buffer.len()); + self.counter = 0; + } + } /// 重置builder @@ -68,7 +107,15 @@ impl BlockBuilder { /// block_builder.reset(); /// ``` pub fn reset(&mut self) { - todo!() + self.buffer.clear(); + + self.restarts.clear(); + // First restart point is at offset 0 + self.restarts.push(0); + + self.counter = 0; + self.finished = false; + self.last_key.clear(); } /// 追加Restart points diff --git a/src/table/block_builder_test.rs b/src/table/block_builder_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..3bec234627d9c749cdee226016723e21cdbc9c64 --- /dev/null +++ b/src/table/block_builder_test.rs @@ -0,0 +1,25 @@ + +mod test { + use crate::table::block_builder::BlockBuilder; + use crate::util::options::OptionsPtr; + + use crate::util::Result; + use crate::util::slice::Slice; + + #[test] + fn test_new() { + let opt = OptionsPtr::default(); + + let block_builder = BlockBuilder::new(opt); + assert_eq!(block_builder.get_restarts().len(), 1); + } + + #[test] + fn test_add() { + let opt = OptionsPtr::default(); + + let mut block_builder = BlockBuilder::new(opt); + block_builder.add(Slice::from("a"), Slice::from("b")); + assert_eq!(block_builder.get_restarts().len(), 1); + } +} \ No newline at end of file diff --git a/src/table/filter_block.rs b/src/table/filter_block.rs index 9f0fb38b1bf21a0ebefc31257ed30baf7f5cff5e..8283afcbe04996814927d568cf8d0b5d3040dfe8 100644 --- a/src/table/filter_block.rs +++ b/src/table/filter_block.rs @@ -1,4 +1,5 @@ use std::io::Write; +use std::ops::{Deref, DerefMut}; use std::sync::Arc; use crate::debug; use crate::traits::filter_policy_trait::{FilterPolicy, FilterPolicyPtr}; @@ -96,6 +97,8 @@ pub trait FilterBlock { fn get_tmp_filter_offsets(&self) -> Vec; } +pub type FilterBlockBuilderPtr = Arc>; + /// SSTable 文件里面的 meta block 构建器, 按内存里面指定的格式整理在内存中 pub struct FilterBlockBuilder { // 指向一个具体的filter_policy diff --git a/src/table/mod.rs b/src/table/mod.rs index 214c79658e8c118e56e3016c63f2f6cb7c08daf9..11e5da1b8e51cce5068ad6bc1e6334c029f1fe74 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -1,5 +1,6 @@ pub mod block; pub mod block_builder; +mod block_builder_test; pub mod filter_block; mod filter_block_test; mod filter_block_test_filter_policy; diff --git a/src/table/table_builder.rs b/src/table/table_builder.rs index c2d449260de3aa60f457c410609d47dd5d7f652d..d74d5b9e6d428cc64f51895166297fdb3dc5cf4c 100644 --- a/src/table/table_builder.rs +++ b/src/table/table_builder.rs @@ -2,7 +2,7 @@ use std::borrow::Borrow; use std::fs::File; use std::sync::Arc; use crate::table::block_builder::BlockBuilder; -use crate::table::filter_block::{FilterBlock, FilterBlockBuilder}; +use crate::table::filter_block::{FilterBlock, FilterBlockBuilder, FilterBlockBuilderPtr}; use crate::table::format::BlockHandle; use crate::traits::filter_policy_trait::FilterPolicy; use crate::util::options::{CompressionType, OptionsPtr, Options}; @@ -42,7 +42,7 @@ struct Rep<> { closed: bool, // 生成 SSTable 中的元数据区域 - filter_block: Option, + filter_block: Option, // 判断是否需要生成 SSTable中的数据索引， SSTable中每次生成一个完整的块之后，需要将该值置为 true， 说明需要为该块添加索引 pending_index_entry: bool, // Handle to add to index block @@ -55,13 +55,16 @@ struct Rep<> { impl TableBuilder { pub fn new_with_writable_file(options: OptionsPtr, writable_file: Arc) -> Self { - let rep = Rep::new(options, writable_file); + let mut rep = Rep::new(options, writable_file); - // Self { - // rep + // todo + // if rep.filter_block.is_some() { + // rep.filter_block.unwrap().as_mut().start_block(0); // } - todo!() + Self { + rep: Box::new(rep) + } } /// 写入 entry @@ -108,14 +111,20 @@ impl TableBuilder { impl Rep { pub fn new(opt: OptionsPtr, writableFile: Arc) -> Self { - let mut filter_block: Option; + let mut filter_block: Option; if opt.filter_policy.is_none() { filter_block = None; }else { - filter_block = Some(FilterBlockBuilder::new_with_policy(opt.filter_policy.clone().unwrap())); + filter_block = Some( + Arc::new(Box::new( + FilterBlockBuilder::new_with_policy(opt.filter_policy.clone().unwrap()) + )) + ); } - // TODo if let sytax - // let filter_block = opt.filter_policy.map(|e|FilterBlockBuilder::new_with_policy(e.clone().unwrap())); + // todo maybe try if let sytax + let filter_block1 = opt.filter_policy.clone().map( + |e| Arc::new(Box::new(FilterBlockBuilder::new_with_policy(e.clone()))) + ); Self { options: opt.clone(), diff --git a/src/traits/comparator_trait.rs b/src/traits/comparator_trait.rs index 477eb58d765ec5b2593fefc739561b1c18168e2e..b28075c413e325e35de2509f5804cacd95926269 100644 --- a/src/traits/comparator_trait.rs +++ b/src/traits/comparator_trait.rs @@ -1,4 +1,6 @@ use std::cmp::Ordering; +use std::todo; +use crate::db::db_format::InternalKeyComparator; use crate::util::slice::Slice; /// 比较器 @@ -85,5 +87,4 @@ pub trait Comparator { /// /// ``` fn find_short_successor(&self, key: &String) -> String; - } \ No newline at end of file diff --git a/src/util/error.rs b/src/util/error.rs index f3d6be25778da0370549b862c5d791b72947b844..ca541773fd9ee75ee70049b497caa6b467878222 100644 --- a/src/util/error.rs +++ b/src/util/error.rs @@ -63,11 +63,12 @@ // /// Provides the `map_err_to_code` method for `Result`. // /// // /// ``` +// /// use level_db_rust::util::error::ErrorCode; // /// use crate::util::error::ErrorCode; // /// use crate::util::error::ToErrorCode; // /// // /// let x: std::result::Result<(), std::fmt::Error> = Err(std::fmt::Error {}); -// /// let y: common_exception::Result<()> = x.map_err_to_code(ErrorCode::UnknownException, || 123); +// /// let y: common_exception::Result<()> = x.map_err_to_code(ErrorCode::OK, || 123); // /// // /// assert_eq!( // /// "Code: 1067, Text = 123, cause: an error occurred when formatting an argument.", diff --git a/src/util/error_code.rs b/src/util/error_code.rs index 9b841669f4f84305c0db3cc90456369f1d6352a8..60e198b418fa0819469ee0bafdb49eb2ab962e9b 100644 --- a/src/util/error_code.rs +++ b/src/util/error_code.rs @@ -34,7 +34,7 @@ // } // } // } -// k +// // build_exceptions! { // Ok(0), // Internal(1001), diff --git a/src/util/error_test.rs b/src/util/error_test.rs index 7dbc8e084585bc9efd86cbc155cbd1c926fbbb4f..2865e8e1b84cf579ec52dd66f0c2ec5d53851884 100644 --- a/src/util/error_test.rs +++ b/src/util/error_test.rs @@ -12,7 +12,7 @@ // ErrorCode::Ok; // let err: StatusError = StatusError::KNotFound("a".to_string()); // let ok_err: StatusError = StatusError::KOk(); -// +// ErrorCode:: // debug!("{:?}", err.borrow()); // // assert_eq!("KNotFound("a")", err.borrow()); // diff --git a/src/util/mod.rs b/src/util/mod.rs index 289ca42b8b08bbf84c0d7db114ae8acc3724ea21..67e03c2042c3d1a4f992fa6fbc75d1f39afe3daa 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -47,9 +47,9 @@ pub mod unsafe_slice; pub mod env; mod env_test; pub mod mem_debug; -// pub mod error; -// mod error_test; -// pub mod error_code; +pub mod error; +mod error_test; +pub mod error_code; /// 定义别名 pub type Result = result::Result;