diff --git a/zstd/BUILD.gn b/zstd/BUILD.gn new file mode 100644 index 0000000000000000000000000000000000000000..99cc7199f5d1f80b91518dda36577dc1ffb3e740 --- /dev/null +++ b/zstd/BUILD.gn @@ -0,0 +1,516 @@ +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build/ohos.gni") + +declare_args() { + enable_zstd_test = false +} + +config("libzstd_config") { + include_dirs = [ + ".", + "./zstd", + "./zstd/lib", + "./zstd/lib/common", + "./zstd/lib/compress", + "./zstd/lib/decompress", + "./zstd/lib/deprecated", + "./zstd/lib/dictBuilder", + "./zstd/lib/legacy", + "../lz4", + "../lz4/lib", + "../zlib", + "../xz/xz/", + "../xz/xz/src/", + "../xz/xz/dos/", + "../xz/xz/src/liblzma/api/", + "../xz/xz/src/liblzma/api/lzma/", + "../xz/xz/src/common/", + "../xz/xz/src/liblzma", + "../xz/xz/src/liblzma/common/", + "../xz/xz/src/liblzma/check", + "../xz/xz/src/lz", + "../xz/xz/src/lzma", + ] + + cflags = [ + "-fPIC", + "-Wall", + "-Wextra", + "-fexceptions", + "-Wno-error=unused-parameter", + "-Wno-error=int-conversion", + "-Wno-error=format", + "-DZSTD_BUILD_CONTRIB", + "-DZSTD_BUILD_STATIC", + "-DZSTD_BUILD_TESTS", + "-DZSTD_ZLIB_SUPPORT", + "-DZSTD_LZMA_SUPPORT", + "-Dzstd_COMPRESSION", + "-Dzstd_DECOMPRESSION", + "-Dzstd_DICTBUILDER", + "-DHAVE_LZ4", + "-DZSTD_LZ4COMPRESS", + "-DZSTD_LZ4DECOMPRESS", + "-DHAVE_PTHREAD", + "-DHAVE_THREAD", + "-DZSTD_MULTITHREAD", + "-DHAVE_ZLIB", + "-DZSTD_GZCOMPRESS", + "-DZSTD_GZDECOMPRESS", + "-DHAVE_LZMA", + "-DZSTD_LZMACOMPRESS", + "-DZSTD_LZMADECOMPRESS", + ] +} +ohos_shared_library("zstd_shared") { + sources = [ + "./zstd/lib/common/debug.c", + "./zstd/lib/common/entropy_common.c", + "./zstd/lib/common/error_private.c", + "./zstd/lib/common/fse_decompress.c", + "./zstd/lib/common/pool.c", + "./zstd/lib/common/threading.c", + "./zstd/lib/common/xxhash.c", + "./zstd/lib/common/zstd_common.c", + "./zstd/lib/compress/fse_compress.c", + "./zstd/lib/compress/hist.c", + "./zstd/lib/compress/huf_compress.c", + "./zstd/lib/compress/zstd_compress.c", + "./zstd/lib/compress/zstd_compress_literals.c", + "./zstd/lib/compress/zstd_compress_sequences.c", + "./zstd/lib/compress/zstd_compress_superblock.c", + "./zstd/lib/compress/zstd_double_fast.c", + "./zstd/lib/compress/zstd_fast.c", + "./zstd/lib/compress/zstd_lazy.c", + "./zstd/lib/compress/zstd_ldm.c", + "./zstd/lib/compress/zstd_opt.c", + "./zstd/lib/compress/zstdmt_compress.c", + "./zstd/lib/decompress/huf_decompress.c", + "./zstd/lib/decompress/zstd_ddict.c", + "./zstd/lib/decompress/zstd_decompress.c", + "./zstd/lib/decompress/zstd_decompress_block.c", + "./zstd/lib/deprecated/zbuff_common.c", + "./zstd/lib/deprecated/zbuff_compress.c", + "./zstd/lib/deprecated/zbuff_decompress.c", + "./zstd/lib/dictBuilder/cover.c", + "./zstd/lib/dictBuilder/divsufsort.c", + "./zstd/lib/dictBuilder/fastcover.c", + "./zstd/lib/dictBuilder/zdict.c", + "./zstd/lib/legacy/zstd_v01.c", + "./zstd/lib/legacy/zstd_v02.c", + "./zstd/lib/legacy/zstd_v03.c", + "./zstd/lib/legacy/zstd_v04.c", + "./zstd/lib/legacy/zstd_v05.c", + "./zstd/lib/legacy/zstd_v06.c", + "./zstd/lib/legacy/zstd_v07.c", + ] + + configs = [ + ":libzstd_config" + ] + + deps = [ + "../lz4:liblz4_static", + "../zlib:libz", + "../xz:libxz", + ] + + part_name = "zstd" +} + + +config ("regression_config") { + include_dirs = [ + "./zstd", + "./zstd/lib/", + "./zstd/programs/", + "./zstd/lib/common/", + "../curl/", + "../curl/include/", + "../curl/include/curl/", + ] + + cflags = [ + "-fPIC", + "-Wall", + "-Wextra", + "-fexceptions", + "-Wno-error=unused-parameter", + "-Wno-error=unused-variable", + "-Wno-error=unused-function", + "-Wno-error=sign-compare", + "-Wno-error=undef", + "-Wno-error=macro-redefined", + "-Wno-error=int-conversion", + "-Wno-error=format", + "-Wno-error=logical-op-parentheses", + "-Wno-error=parentheses", + "-Wno-error=deprecated-declarations", + "-Wno-error=missing-field-initializers", + ] + + ldflags = [ + "-lpthread", + ] +} + +ohos_executable("zst_test_regression") { + sources = [ + "./zstd/tests/regression/config.c", + "./zstd/tests/regression/data.c", + "./zstd/tests/regression/method.c", + "./zstd/tests/regression/result.c", + "./zstd/tests/regression/test.c", + "./zstd/programs/util.c", + "./zstd/lib/common/xxhash.c", + ] + + public_configs = [ ":regression_config" ] + + deps = [ + ":zstd_shared", + "../curl:curl_target" + ] + + defines = [ + ] + + part_name = "zstd" +} + + + +config ("zstd_tests_config") { + include_dirs = [ + "./zstd", + "./zstd/lib", + "./zstd/examples", + ] + + cflags = [ + "-fPIC", + "-Wall", + "-Wextra", + "-fexceptions", + "-Wno-error=unused-parameter", + "-Wno-error=unused-variable", + "-Wno-error=unused-function", + "-Wno-error=sign-compare", + "-Wno-error=undef", + "-Wno-error=macro-redefined", + "-Wno-error=int-conversion", + "-Wno-error=format", + "-Wno-error=logical-op-parentheses", + "-Wno-error=parentheses", + ] +} + +ohos_executable("dictionary_compression") { + sources = [ + "./zstd/examples/dictionary_compression.c", + ] + + public_configs = [ ":zstd_tests_config" ] + + deps = [ + ":zstd_shared", + ] + + defines = [ + ] + + part_name = "zstd" +} + + +ohos_executable("dictionary_decompression") { + sources = [ + "./zstd/examples/dictionary_decompression.c", + ] + + public_configs = [ ":zstd_tests_config" ] + + deps = [ + ":zstd_shared", + ] + + defines = [ + ] + + part_name = "zstd" +} + + +ohos_executable("multiple_simple_compression") { + sources = [ + "./zstd/examples/multiple_simple_compression.c", + ] + + public_configs = [ ":zstd_tests_config" ] + + deps = [ + ":zstd_shared", + ] + + defines = [ + ] + + part_name = "zstd" +} + + +ohos_executable("multiple_streaming_compression") { + sources = [ + "./zstd/examples/multiple_streaming_compression.c", + ] + + public_configs = [ ":zstd_tests_config" ] + + deps = [ + ":zstd_shared", + ] + + defines = [ + ] + + part_name = "zstd" +} + + +ohos_executable("simple_compression") { + sources = [ + "./zstd/examples/simple_compression.c", + ] + + public_configs = [ ":zstd_tests_config" ] + + deps = [ + ":zstd_shared", + ] + + defines = [ + ] + + part_name = "zstd" +} + + +ohos_executable("simple_decompression") { + sources = [ + "./zstd/examples/simple_decompression.c", + ] + + public_configs = [ ":zstd_tests_config" ] + + deps = [ + ":zstd_shared", + ] + + defines = [ + ] + + part_name = "zstd" +} + +ohos_executable("streaming_compression_thread_pool") { + sources = [ + "./zstd/examples/streaming_compression_thread_pool.c", + ] + + public_configs = [ ":zstd_tests_config" ] + + deps = [ + ":zstd_shared", + ] + + defines = [ + ] + + part_name = "zstd" +} + + +ohos_executable("streaming_compression") { + sources = [ + "./zstd/examples/streaming_compression.c", + ] + + public_configs = [ ":zstd_tests_config" ] + + deps = [ + ":zstd_shared", + ] + + defines = [ + ] + + part_name = "zstd" +} + + +ohos_executable("streaming_decompression") { + sources = [ + "./zstd/examples/streaming_decompression.c", + ] + + public_configs = [ ":zstd_tests_config" ] + + deps = [ + ":zstd_shared", + ] + + defines = [ + ] + + part_name = "zstd" +} + + +ohos_executable("streaming_memory_usage") { + sources = [ + "./zstd/examples/streaming_memory_usage.c" + ] + + public_configs = [ ":zstd_tests_config" ] + + deps = [ + ":zstd_shared", + ] + + defines = [ + ] + + part_name = "zstd" +} + + +config ("zstd_programs_config") { + include_dirs = [ + "./zstd", + "./zstd/programs", + "./zstd/lib", + "./zstd/zlibWrapper", + "../lz4", + "../lz4/lib", + "../zlib", + "../xz/xz/", + "../xz/xz/src/", + "../xz/xz/dos/", + "../xz/xz/src/liblzma/api/", + "../xz/xz/src/liblzma/api/lzma/", + "../xz/xz/src/common/", + "../xz/xz/src/liblzma", + "../xz/xz/src/liblzma/common/", + "../xz/xz/src/liblzma/check", + "../xz/xz/src/lz", + "../xz/xz/src/lzma", + ] + + cflags = [ + "-fPIC", + "-Wall", + "-Wextra", + "-fexceptions", + "-Wno-error=unused-parameter", + "-Wno-error=unused-variable", + "-Wno-error=sign-compare", + "-Wno-error=undef", + "-Wno-error=macro-redefined", + "-Wno-error=int-conversion", + "-Wno-error=format", + "-Wno-error=logical-op-parentheses", + "-Wno-error=parentheses", + "-Wno-error=implicit-function-declaration", + "-DHAVE_LZ4", + "-DZSTD_LZ4COMPRESS", + "-DZSTD_LZ4DECOMPRESS", + "-DHAVE_PTHREAD", + "-DHAVE_THREAD", + "-DZSTD_MULTITHREAD", + "-DHAVE_ZLIB", + "-DZSTD_GZCOMPRESS", + "-DZSTD_GZDECOMPRESS", + "-DHAVE_LZMA", + "-DZSTD_LZMACOMPRESS", + "-DZSTD_LZMADECOMPRESS", + ] + + ldflags = [ + "-lpthread", + ] +} + +ohos_executable("zstd") { + sources = [ + "./zstd/programs/benchfn.c", + "./zstd/programs/benchzstd.c", + "./zstd/programs/datagen.c", + "./zstd/programs/dibio.c", + "./zstd/programs/fileio.c", + "./zstd/programs/timefn.c", + "./zstd/programs/util.c", + "./zstd/programs/zstdcli.c", + "./zstd/programs/zstdcli_trace.c", + ] + + public_configs = [ ":zstd_programs_config" ] + + deps = [ + ":zstd_shared", + "../lz4:liblz4_static", + "../zlib:libz", + "../xz:libxz", + ] + + defines = [ + "PACKAGE_VERSION", + ] + + part_name = "zstd" +} + +group("examples") { + if (enable_zstd_test) { + deps = [ + ":dictionary_compression", + ":dictionary_decompression", + ":multiple_simple_compression", + ":multiple_streaming_compression", + ":simple_compression", + ":simple_decompression", + ":streaming_compression_thread_pool", + ":streaming_compression", + ":streaming_decompression", + ":streaming_memory_usage", + ] + } +} + +group("zstd_exe") { + if (enable_zstd_test) { + deps = [ + ":zstd" + ] + } +} + +group("zst_regression") { + if (enable_zstd_test) { + deps = [ + ":zst_test_regression" + ] + } +} + diff --git a/zstd/README.OpenSource b/zstd/README.OpenSource new file mode 100644 index 0000000000000000000000000000000000000000..f6786dae8ec4580f11426bf4a024408ff0377021 --- /dev/null +++ b/zstd/README.OpenSource @@ -0,0 +1,11 @@ +[ + { + "Name": "zstd", + "License": "BSD and GPLv2", + "License File": "COPYRIGHT", + "Version Number": "v1.5.2", + "Owner": "caiqi14@huawei.com", + "Upstream URL": "https://github.com/facebook/zstd", + "Description": "Zstandard, or zstd as short version, is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level and better compression ratios. " + } +] \ No newline at end of file diff --git a/zstd/README_zh.md b/zstd/README_zh.md new file mode 100644 index 0000000000000000000000000000000000000000..87a7a870cb97b4918401d396354a812af3a319fa --- /dev/null +++ b/zstd/README_zh.md @@ -0,0 +1,10 @@ +# zstd 三方库说明 +## 功能简介 +zstd 是一种快速的无损压缩算法，是针对 zlib 级别的实时压缩方案，以及更好的压缩比。 +## 使用约束 +- ROM版本：OpenHarmony3.2 Beta1 +- 三方库版本：v1.5.2 +- 当前适配的功能：完成了生成和解码 .zst 格式以及字典压缩、解压缩 +- [License : BSD and GPLv2](https://github.com/facebook/zstd/tree/zstd-0.4.2) +## 集成方式 ++ [系统Rom包集成](docs/rom_integrate.md) \ No newline at end of file diff --git a/zstd/bundle.json b/zstd/bundle.json new file mode 100644 index 0000000000000000000000000000000000000000..d9d310175650bb98559d2c033717b978d998bce3 --- /dev/null +++ b/zstd/bundle.json @@ -0,0 +1,33 @@ +{ + "name": "@ohos/zstd_lib", + "description": "Zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level compression ratio.", + "version": "3.1", + "license": "BSD and GPLv2", + "publishAs": "code-segment", + "segment": { + "destPath": "third_party/zstd" + }, + "dirs": {}, + "scripts": {}, + "readmePath": { + "en": "README" + }, + "component": { + "name": "zstd_lib", + "subsystem": "thirdparty", + "syscap": [], + "features": [], + "adapted_system_type": [], + "rom": "", + "ram": "", + "deps": { + "components": [], + "third_party": [] + }, + "build": { + "sub_component": ["//third_party/zstd:zstd_shared","//third_party/zstd:examples","//third_party/zstd:zstd_exe","//third_party/zstd:zst_regression"], + "inner_kits": [], + "test": [] + } + } +} \ No newline at end of file diff --git a/zstd/docs/pic/results.png b/zstd/docs/pic/results.png new file mode 100644 index 0000000000000000000000000000000000000000..d42365e818698ed10c51e62a5a4137cbbc24aed7 Binary files /dev/null and b/zstd/docs/pic/results.png differ diff --git a/zstd/docs/rom_integrate.md b/zstd/docs/rom_integrate.md new file mode 100644 index 0000000000000000000000000000000000000000..e5b28e03d5aa8080b2af6c3631aac3d5c7604cd2 --- /dev/null +++ b/zstd/docs/rom_integrate.md @@ -0,0 +1,108 @@ +# zstd如何集成到系统Rom +## 准备源码工程 +本库是基于OpenHarmony-v3.2-Beta1版本，在润和RK3568开发板上验证的。如果是从未使用过RK3568，可以先查看[润和RK3568开发板标准系统快速上手](https://gitee.com/openharmony-sig/knowledge_demo_temp/tree/master/docs/rk3568_helloworld)。 +## 准备系统Rom源码 +源码获取方法请参照：[OpenHarmony源码下载](https://gitee.com/openharmony/docs/blob/OpenHarmony-v3.2-Beta1/zh-cn/release-notes/OpenHarmony-v3.2-beta1.md#%E6%BA%90%E7%A0%81%E8%8E%B7%E5%8F%96) +## 增加构建脚本及配置文件 +- 下载本仓库代码 + ``` + cd ~/ + git clone git@gitee.com:openharmony-sig/tpc_c_cplusplus.git --depth=1 + ``` +- 三方库目录结构 + ``` + tpc_c_cplusplus/zstd + |-- docs # 存放三方库相关文档的文件夹 + |-- BUILD.gn # 构建脚本，支持rom包集成 + |-- bundle.json # 三方库组件定义文件 + ├── README.OpenSource # 说明三方库源码的下载地址，版本，license等信息 + ├── README_zh.md + ``` +- 将本仓库文件夹拷贝到third_party下 + ``` + cp ~/tpc_c_cplusplus/zstd ~/openharmony/third_party/ -rf + ``` +## 准备三方库源码 +- 将源码下载到zstd目录并将其解压出来。 +``` +cd ~/openharmony/third_party/zstd # 进入到zstd目录 +git clone -b v1.5.2 https://github.com/facebook/zstd.git # 下载三方库 +``` +## 系统Rom中引入三方库 +准备完三方库代码后，我们需要将三方库加入到编译构建体系中。标准系统编译构建可以参考文档[标准系统编译构建指导](https://gitee.com/openharmony/docs/blob/OpenHarmony-3.2-Beta1/zh-cn/device-dev/subsystems/subsys-build-standard-large.md)。 +我们默认三方库是属于OpenHarmony的thirdparty子系统，如果需要自己定义子系统参考文档[如何为三方库组件中添加一个三方库](https://gitee.com/openharmony-sig/knowledge/blob/master/docs/openharmony_getstarted/port_thirdparty/README.md) +在OpenHarmony源码的vendor/hihope/rk3568/config.json文件中，新增需要编译的组件，如下： +``` +{ + "subsystem": "thirdparty", + "components": [ + { + "component": "musl", + "features": [] + }, + { + "component": "zstd", + "features": [] + } + ] +} +``` +## 系统Rom中引入三方库测试程序 +在OpenHarmony源码的vendor/hihope/rk3568/config.json文件,对应组件的features中打开编译选项，如下： +``` +{ + "subsystem": "thirdparty", + "components": [ + { + "component": "musl", + "features": [] + }, + { + "component": "zstd", + "features": ["enable_zstd_test=true"] + } + ] +} +``` +## 编译工程 +在OpenHarmony源码根目录下 +``` +cd ~/openharmony +``` +- 选择产品 +``` +hb set #该命令会列出所有可选平台，这里我们选择rk3568 +``` +- 运行编译 +``` +hb build --target-cpu arm64 #编译64位系统使用:arm64， 编译32位系统使用:arm +``` +- 生成文件的路径，可执行文件和库文件都在out/rk3568/thirdparty/zstd目录下，同时也打包到了镜像中 +## 安装应用程序 +将编译生成的库和测试文件放到板子上运行，为避免每次将文件推入设备都烧录整个镜像，我们使用hdc_std工具将文件推到开发板上 +- 首先，将hdc_std工具编译出来 + ``` + hb set #源码根目录下使用hb set 选择产品ohos-sdk + hb build #然后编译，工具编译出来在out/sdk/ohos-sdk/windows/toolchains/hdc_std.exe + ``` +- 然后，将工具拷贝到Windows，可以为工具目录配置环境变量，也可以在工具所在目录打开windows命令行 +- 最后，将原生库测试需要的所有文件打包成zstd.tar,并拷贝到windows下 +- 将文件推送到开发板，在windows命令行进行如下操作 + ``` + hdc_std shell mount -o remount,rw / #修改系统权限为可读写 + hdc_std file send zstd.tar / #将文件包推入开发板 + hdc_std shell #进入开发板 + tar xvf zstd.tar #解压 + cd zstd #进入zstd 目录 + mv libzstd_shared.z.so /system/lib64/ #64位系统需要将库文件拷贝到系统lib64目录, 32位系统则是lib目录 + ``` +## 运行效果 +由于测试用例比较多，这里演示常用的.zst文件压缩和解压缩功能，并将运行结果和期望结果做对比，内容一致，如下图： + ![results](pic/results.png) +## 参考资料 +- [润和RK3568开发板标准系统快速上手](https://gitee.com/openharmony-sig/knowledge_demo_temp/tree/master/docs/rk3568_helloworld) +- [OpenHarmony三方库地址](https://gitee.com/openharmony-tpc) +- [OpenHarmony知识体系](https://gitee.com/openharmony-sig/knowledge) +- [OpenHarmony源码下载](https://gitee.com/openharmony/docs/blob/OpenHarmony-v3.2-Beta1/zh-cn/release-notes/OpenHarmony-v3.2-beta1.md#%E6%BA%90%E7%A0%81%E8%8E%B7%E5%8F%96) +- [标准系统编译构建指导](https://gitee.com/openharmony/docs/blob/OpenHarmony-3.2-Beta1/zh-cn/device-dev/subsystems/subsys-build-standard-large.md) +- [如何为三方库组件中添加一个三方库](https://gitee.com/openharmony-sig/knowledge/blob/master/docs/openharmony_getstarted/port_thirdparty/README.md) \ No newline at end of file