From ae767e1db9413363844c8081abd1b994bcbb54c6 Mon Sep 17 00:00:00 2001 From: wangboo <5417808+wangboa@user.noreply.gitee.com> Date: Mon, 12 Dec 2022 17:42:12 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Arena=E5=AE=9E=E7=8E=B0,?= =?UTF-8?q?=20DataPtr=E5=AE=9A=E4=B9=89,=20Encoding=20=E7=89=B9=E8=B4=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/arena.rs | 36 +++++++++++++++++ src/util/coding.rs | 58 +++++++-------------------- src/util/coding_test.rs | 16 -------- src/util/data_ptr.rs | 84 +++++++++++++++++++++++++++++++++++++++ src/util/data_ptr_test.rs | 14 +++++++ src/util/mod.rs | 5 ++- 6 files changed, 153 insertions(+), 60 deletions(-) create mode 100644 src/util/arena.rs create mode 100644 src/util/data_ptr.rs create mode 100644 src/util/data_ptr_test.rs diff --git a/src/util/arena.rs b/src/util/arena.rs new file mode 100644 index 0000000..7263060 --- /dev/null +++ b/src/util/arena.rs @@ -0,0 +1,36 @@ +use std::ptr::NonNull; +use crate::util::data_ptr::DataPtr; + +// Arena block size +const ARENA_BLOCK_SIZE: usize = 4096; + +struct Arena { + +} + +impl Arena { + + pub fn allocate(_bytes: usize) -> NonNull { + todo!() + } + + pub fn allocate_align(_bytes: usize) -> NonNull { + todo!() + } + + fn allocate_fallback(_bytes: usize) -> DataPtr { + todo!() + } + + fn allocate_new_block(_block_size: usize) -> DataPtr { + todo!() + } + +} + +impl Drop for Arena { + + fn drop(&mut self) { + todo!() + } +} \ No newline at end of file diff --git a/src/util/coding.rs b/src/util/coding.rs index 74f2492..99a5c67 100644 --- a/src/util/coding.rs +++ b/src/util/coding.rs @@ -1,47 +1,19 @@ -const B: u8 = 128; +use std::mem::size_of; -/// -/// -/// -pub trait Coding { - fn varint32(self, buf: &mut [u8]); -} +use crate::util::data_ptr::DataPtr; -impl Coding for u32 -{ - fn varint32(self, buf: &mut [u8]) { - // if self < 1 << 7 { - // buf[0] = self as u8; - // } else if self < 1 << 14 { - // buf[0] = self | B; - // println!("{:b}", buf[0]); - // buf[1] = buf[0] >> 7; - // println!("{:b}", buf[1]); - // } else if self < 1 << 21 { - // buf[0] = (self | B as u8) as u8; - // println!("{:b}", buf[0]); - // buf[1] = (buf[0] >> 7) | B; - // println!("{:b}", buf[1]); - // buf[2] = buf[1] >> 14; - // println!("{:b}", buf[2]); - // } else if self < 1 << 28 { - // buf[0] = (self | B as u8) as u8; - // println!("{:b}", buf[0]); - // buf[1] = (buf[0] >> 7) | B; - // println!("{:b}", buf[1]); - // buf[2] = (buf[1] >> 14) | B; - // println!("{:b}", buf[2]); - // buf[3] = buf[2] >> 21; - // } else { - // buf[0] = (self | B as u8) as u8; - // println!("{:b}", buf[0]); - // buf[1] = (buf[0] >> 7) | B; - // println!("{:b}", buf[1]); - // buf[2] = (buf[1] >> 14) | B; - // println!("{:b}", buf[2]); - // buf[3] = (buf[2] >> 21) | B; - // println!("{:b}", buf[3]); - // buf[4] = buf[3] >> 28; - // } +/// 编解码特性 +/// LevelDB需要支持的基本类型都需要实现此特质, 如 u8, i8, u32 等 +pub trait Coding { + + fn write(self, ptr: DataPtr) -> DataPtr; + + fn read(ptr: DataPtr) -> T; + + fn read_once(ptr: DataPtr) -> (T, DataPtr) { + let value: T = Self::read(ptr); + unsafe { + (value, ptr.offset(size_of::() as isize)) + } } } \ No newline at end of file diff --git a/src/util/coding_test.rs b/src/util/coding_test.rs index 3c4184f..ed03565 100644 --- a/src/util/coding_test.rs +++ b/src/util/coding_test.rs @@ -1,20 +1,4 @@ mod test { use crate::util::coding::Coding; - #[test] - fn test_i32() { - let mut buf: [u8; 8] = [0, 0, 0, 0, 0, 0, 0, 0]; - let x = 127.varint32(&mut buf); - println!("{:?}", buf); - println!("{:?}", x); - println!("{:b}", buf[0]); - } - - #[test] - fn test_offset() { - let buf: [u8; 3] = [0, 1, 2]; - let mut offset: usize = 0; - offset += 1; - println!("{:?}", buf[offset]) - } } diff --git a/src/util/data_ptr.rs b/src/util/data_ptr.rs new file mode 100644 index 0000000..76c2245 --- /dev/null +++ b/src/util/data_ptr.rs @@ -0,0 +1,84 @@ +use std::mem; +use std::mem::size_of; + +#[derive(Copy, Clone)] +pub struct DataPtr { + ptr: *mut u8, +} + +impl From<*mut u8> for DataPtr { + #[inline] + fn from(ptr: *mut u8) -> Self { + Self { + ptr + } + } +} + +impl From<*const u8> for DataPtr { + #[inline] + fn from(ptr: *const u8) -> Self { + Self { + ptr: ptr as *mut u8 + } + } +} + +impl DataPtr { + /// 向 DataPtr 中写入一个值 + /// + /// # Unsafe + /// 调用方需要保证写入数据的大小不会超过 DataPtr 剩余可用的内存大小 + /// + /// # Arguments + /// + /// * `value`: 要被写入的值 + /// + /// returns: DataPtr 写入后的指针偏移量 + /// + /// # Examples + /// + /// ``` + /// use std::alloc::{alloc, Layout}; + /// let raw_ptr = unsafe { + /// alloc(Layout::from_size_align(8, 8).unwrap()) + /// }; + /// let ptr = DataPtr::from(raw_ptr); + /// unsafe { + /// // write multi values + /// ptr.write(18_u32).write(32_u32); + /// } + /// ``` + #[inline] + pub unsafe fn write(&self, value: T) -> DataPtr + where T: Sized { + unsafe { + (self.ptr as *mut _ as *mut T).write(value); + self.ptr.offset(size_of::() as isize).into() + } + } + + + /// 让 DataPtr 的指针偏移指定长度 + /// + /// # Unsafe + /// 调用方必须保证增加额外得偏移长度不会造成指针越界 + /// + /// # Arguments + /// + /// * `offset`: 偏移长度 + /// + /// returns: DataPtr + /// + /// # Examples + /// + /// ``` + /// + /// ``` + #[inline] + pub unsafe fn offset(&self, offset: isize) -> DataPtr { + self.ptr.offset(offset).into() + } + + +} \ No newline at end of file diff --git a/src/util/data_ptr_test.rs b/src/util/data_ptr_test.rs new file mode 100644 index 0000000..940b63d --- /dev/null +++ b/src/util/data_ptr_test.rs @@ -0,0 +1,14 @@ +use std::alloc::{alloc, Layout}; +use crate::util::data_ptr::DataPtr; + +#[test] +fn test_create() { + let raw_ptr = unsafe { + alloc(Layout::from_size_align(8, 8).unwrap()) + }; + let ptr = DataPtr::from(raw_ptr); + unsafe { + ptr.write(18_u32); + ptr.write(32_u32); + } +} \ No newline at end of file diff --git a/src/util/mod.rs b/src/util/mod.rs index fe7c884..2703933 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -2,4 +2,7 @@ pub mod slice; pub mod status; mod slice_test; pub mod coding; -mod coding_test; \ No newline at end of file +mod coding_test; +pub mod arena; +pub mod data_ptr; +mod data_ptr_test; \ No newline at end of file -- Gitee From 4ef960e302f563a129642d009d9a9b5a78069360 Mon Sep 17 00:00:00 2001 From: wangboo <5417808+wangboa@user.noreply.gitee.com> Date: Mon, 12 Dec 2022 20:05:05 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E5=A2=9E=E5=8A=A0arena=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/arena.rs | 94 ++++++++++++++++++++++++++++++++++----- src/util/arena_test.rs | 35 +++++++++++++++ src/util/coding.rs | 18 +------- src/util/data_ptr.rs | 84 ---------------------------------- src/util/data_ptr_test.rs | 14 ------ src/util/mod.rs | 5 ++- 6 files changed, 121 insertions(+), 129 deletions(-) create mode 100644 src/util/arena_test.rs delete mode 100644 src/util/data_ptr.rs delete mode 100644 src/util/data_ptr_test.rs diff --git a/src/util/arena.rs b/src/util/arena.rs index 7263060..d9b3f95 100644 --- a/src/util/arena.rs +++ b/src/util/arena.rs @@ -1,36 +1,106 @@ +use std::alloc::{alloc, dealloc, Layout}; use std::ptr::NonNull; -use crate::util::data_ptr::DataPtr; +use std::slice; // Arena block size const ARENA_BLOCK_SIZE: usize = 4096; -struct Arena { +pub struct Arena { + alloc_ptr: Option>, + alloc_bytes_remaining: usize, + blocks: Vec<(NonNull, Layout)>, + memory_usage: usize, +} +impl Default for Arena { + fn default() -> Self { + Self { + alloc_ptr: None, + alloc_bytes_remaining: 0, + blocks: vec![], + memory_usage: 0, + } + } } impl Arena { - pub fn allocate(_bytes: usize) -> NonNull { - todo!() + /// 申请一块内存 + /// + /// # Arguments + /// + /// * `bytes`: 申请内存大小(byte) + /// + /// returns: &mut [u8] + /// 内存的 byte 数组 + /// # Examples + /// + /// ``` + ///let arena = Arena::default(); + /// // 申请 12 字节大小的内存 + /// let buf = arena.allocate(12); + /// ``` + #[inline] + pub fn allocate(&mut self, bytes: usize) -> &mut [u8] { + self.allocate_align(bytes, 1) } - pub fn allocate_align(_bytes: usize) -> NonNull { - todo!() + pub fn allocate_align(&mut self, bytes: usize, align: usize) -> &mut [u8] { + if bytes <= self.alloc_bytes_remaining { + self.alloc_bytes_remaining -= bytes; + let result = unsafe { slice::from_raw_parts_mut(self.alloc_ptr.unwrap().as_ptr(), bytes) }; + unsafe { + let new_ptr = self.alloc_ptr.unwrap().as_ptr().offset(bytes as isize); + self.alloc_ptr = Some(NonNull::new_unchecked(new_ptr)); + }; + return result; + } + return self.allocate_fallback(bytes, align); } - fn allocate_fallback(_bytes: usize) -> DataPtr { - todo!() + pub fn memory_usage(&self) -> usize { + self.memory_usage } - fn allocate_new_block(_block_size: usize) -> DataPtr { - todo!() + fn allocate_fallback(&mut self, bytes: usize, align: usize) -> &mut [u8] { + if bytes > ARENA_BLOCK_SIZE / 4 { + unsafe { + let layout = Layout::from_size_align_unchecked(bytes, align); + return self.allocate_new_block(layout); + } + } + unsafe { + self.alloc_bytes_remaining = ARENA_BLOCK_SIZE - bytes; + let layout = Layout::from_size_align_unchecked(ARENA_BLOCK_SIZE, align); + let new_block = self.allocate_new_block(layout); + unsafe { + let ptr = new_block.as_ptr() as *mut u8; + let result = slice::from_raw_parts_mut(ptr, bytes); + self.alloc_ptr = Some(NonNull::new_unchecked(ptr.offset(bytes as isize))); + result + } + } } + + /// 分配一块新的内存 + fn allocate_new_block(&mut self, layout: Layout) -> &mut [u8] { + unsafe { + let data = alloc(layout); + self.memory_usage += layout.size(); + self.blocks.push((NonNull::new_unchecked(data), layout)); + slice::from_raw_parts_mut(data, layout.size()) + } + } } impl Drop for Arena { - + /// 释放内存 fn drop(&mut self) { - todo!() + for (block, layout) in self.blocks.iter() { + unsafe { + dealloc(block.as_ptr(), *layout) + } + } } } \ No newline at end of file diff --git a/src/util/arena_test.rs b/src/util/arena_test.rs new file mode 100644 index 0000000..dbc8cf1 --- /dev/null +++ b/src/util/arena_test.rs @@ -0,0 +1,35 @@ +use crate::util::Arena; + +#[test] +fn test_memory_usage() { + let mut arena = Arena::default(); + let _buf0 = arena.allocate(12); + let _buf1 = arena.allocate(16); + assert_eq!(4096, arena.memory_usage()); + let _buf2 = arena.allocate(3900); + assert_eq!(4096, arena.memory_usage()); + let _buf3 = arena.allocate(1200); + assert_eq!(4096 + 1200, arena.memory_usage()); +} + +#[test] +fn test_allocate() { + let mut arena = Arena::default(); + for i in 0..=12 { + let byte_size = 1 << i; + let buf = arena.allocate(byte_size); + assert_eq!(byte_size, buf.len()); + } + assert_eq!(8192, arena.memory_usage()); +} + +#[test] +fn test_allocate_align() { + let mut arena = Arena::default(); + for i in 0..=12 { + let byte_size = 1 << i; + let buf = arena.allocate_align(byte_size, 8); + assert_eq!(byte_size, buf.len()); + } + assert_eq!(4096 * 2, arena.memory_usage()); +} \ No newline at end of file diff --git a/src/util/coding.rs b/src/util/coding.rs index 99a5c67..1170950 100644 --- a/src/util/coding.rs +++ b/src/util/coding.rs @@ -1,19 +1,3 @@ -use std::mem::size_of; +pub struct Coding { -use crate::util::data_ptr::DataPtr; - -/// 编解码特性 -/// LevelDB需要支持的基本类型都需要实现此特质, 如 u8, i8, u32 等 -pub trait Coding { - - fn write(self, ptr: DataPtr) -> DataPtr; - - fn read(ptr: DataPtr) -> T; - - fn read_once(ptr: DataPtr) -> (T, DataPtr) { - let value: T = Self::read(ptr); - unsafe { - (value, ptr.offset(size_of::() as isize)) - } - } } \ No newline at end of file diff --git a/src/util/data_ptr.rs b/src/util/data_ptr.rs deleted file mode 100644 index 76c2245..0000000 --- a/src/util/data_ptr.rs +++ /dev/null @@ -1,84 +0,0 @@ -use std::mem; -use std::mem::size_of; - -#[derive(Copy, Clone)] -pub struct DataPtr { - ptr: *mut u8, -} - -impl From<*mut u8> for DataPtr { - #[inline] - fn from(ptr: *mut u8) -> Self { - Self { - ptr - } - } -} - -impl From<*const u8> for DataPtr { - #[inline] - fn from(ptr: *const u8) -> Self { - Self { - ptr: ptr as *mut u8 - } - } -} - -impl DataPtr { - /// 向 DataPtr 中写入一个值 - /// - /// # Unsafe - /// 调用方需要保证写入数据的大小不会超过 DataPtr 剩余可用的内存大小 - /// - /// # Arguments - /// - /// * `value`: 要被写入的值 - /// - /// returns: DataPtr 写入后的指针偏移量 - /// - /// # Examples - /// - /// ``` - /// use std::alloc::{alloc, Layout}; - /// let raw_ptr = unsafe { - /// alloc(Layout::from_size_align(8, 8).unwrap()) - /// }; - /// let ptr = DataPtr::from(raw_ptr); - /// unsafe { - /// // write multi values - /// ptr.write(18_u32).write(32_u32); - /// } - /// ``` - #[inline] - pub unsafe fn write(&self, value: T) -> DataPtr - where T: Sized { - unsafe { - (self.ptr as *mut _ as *mut T).write(value); - self.ptr.offset(size_of::() as isize).into() - } - } - - - /// 让 DataPtr 的指针偏移指定长度 - /// - /// # Unsafe - /// 调用方必须保证增加额外得偏移长度不会造成指针越界 - /// - /// # Arguments - /// - /// * `offset`: 偏移长度 - /// - /// returns: DataPtr - /// - /// # Examples - /// - /// ``` - /// - /// ``` - #[inline] - pub unsafe fn offset(&self, offset: isize) -> DataPtr { - self.ptr.offset(offset).into() - } - - -} \ No newline at end of file diff --git a/src/util/data_ptr_test.rs b/src/util/data_ptr_test.rs deleted file mode 100644 index 940b63d..0000000 --- a/src/util/data_ptr_test.rs +++ /dev/null @@ -1,14 +0,0 @@ -use std::alloc::{alloc, Layout}; -use crate::util::data_ptr::DataPtr; - -#[test] -fn test_create() { - let raw_ptr = unsafe { - alloc(Layout::from_size_align(8, 8).unwrap()) - }; - let ptr = DataPtr::from(raw_ptr); - unsafe { - ptr.write(18_u32); - ptr.write(32_u32); - } -} \ No newline at end of file diff --git a/src/util/mod.rs b/src/util/mod.rs index 2703933..567d645 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -4,5 +4,6 @@ mod slice_test; pub mod coding; mod coding_test; pub mod arena; -pub mod data_ptr; -mod data_ptr_test; \ No newline at end of file +mod arena_test; + +pub use arena::Arena; \ No newline at end of file -- Gitee