diff --git a/services/cloudsyncservice/BUILD.gn b/services/cloudsyncservice/BUILD.gn index 900d4ae4b35392c4ddc7871ee6a8a699bd8fc198..d59dba66057b6f63f212322c974676b21b8d838a 100644 --- a/services/cloudsyncservice/BUILD.gn +++ b/services/cloudsyncservice/BUILD.gn @@ -58,7 +58,10 @@ ohos_shared_library("cloudsync_sa") { "LOG_TAG=\"CLOUDSYNC_SA\"", ] - deps = [ "${utils_path}:libdistributedfileutils" ] + deps = [ + "${utils_path}:libdistributedfiledentry", + "${utils_path}:libdistributedfileutils", + ] external_deps = [ "ability_base:want", diff --git a/utils/BUILD.gn b/utils/BUILD.gn index dbe6e84b2a54bf677249b1dc5f6881795093b19f..b177a5d8f214f401d83f1e3e6fdf263d1e1a8a61 100644 --- a/utils/BUILD.gn +++ b/utils/BUILD.gn @@ -83,3 +83,27 @@ ohos_shared_library("libdistributedfileutils") { part_name = "dfs_service" subsystem_name = "filemanagement" } + +config("dentry_public_config") { + include_dirs = [ "dentry/include" ] +} + +ohos_static_library("libdistributedfiledentry") { + include_dirs = [ + "dentry/include", + "log/include", + ] + sources = [ + "dentry/src/file_utils.cpp", + "dentry/src/meta_file.cpp", + ] + + public_configs = [ ":dentry_public_config" ] + + deps = [ "${utils_path}:libdistributedfileutils" ] + + external_deps = [ "c_utils:utils" ] + + part_name = "dfs_service" + subsystem_name = "filemanagement" +} diff --git a/utils/dentry/include/file_utils.h b/utils/dentry/include/file_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..34bd3ae925032bf741aaa2a4573e9251c10ecb2d --- /dev/null +++ b/utils/dentry/include/file_utils.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OHOS_FILEMGMT_DENTRY_FILE_UTILS_H +#define OHOS_FILEMGMT_DENTRY_FILE_UTILS_H + +#include +#include + +namespace OHOS { +namespace FileManagement { +class FileUtils { +public: + FileUtils() = delete; + ~FileUtils() = delete; + + static int32_t Stat(int fd, struct stat &s); + static int32_t TruncateFile(int fd, size_t size); + static int64_t ReadFile(int fd, off_t offset, size_t size, void *data); + static int64_t WriteFile(int fd, const void *data, off_t offset, size_t size); +}; +} // namespace FileManagement +} // namespace OHOS + +#endif diff --git a/utils/dentry/include/meta_file.h b/utils/dentry/include/meta_file.h new file mode 100644 index 0000000000000000000000000000000000000000..f1586d8255fe244be34976892fa88fcb34355153 --- /dev/null +++ b/utils/dentry/include/meta_file.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OHOS_FILEMGMT_DENTRY_META_FILE_H +#define OHOS_FILEMGMT_DENTRY_META_FILE_H + +#include +#include +#include + +#include "unique_fd.h" + +namespace OHOS { +namespace FileManagement { + +struct MetaBase; +class MetaFile { +public: + MetaFile() = delete; + ~MetaFile(); + explicit MetaFile(uint32_t userId, const std::string &path); + + int32_t DoCreate(const MetaBase &base); + int32_t DoRemove(const MetaBase &base); + int32_t DoUpdate(const MetaBase &base); + int32_t DoRename(MetaBase &oldbase, MetaBase &newbase); + int32_t DoLookup(MetaBase &base); + +private: + UniqueFd fd_{}; +}; + +struct MetaBase { + explicit MetaBase(const std::string &name) : name(name) {} + MetaBase() = default; + uint64_t ino{0}; + uint64_t mtime{0}; + uint64_t size{0}; + uint32_t mode{0}; + std::string name{}; + std::string cloudId{}; +}; + +struct BitOps { + static const int BIT_PER_BYTE = 8; + static int TestBit(int nr, const uint8_t addr[]) + { + return 1 & (addr[nr / BIT_PER_BYTE] >> (nr & (BIT_PER_BYTE - 1))); + } + + static void ClearBit(int nr, uint8_t addr[]) + { + addr[nr / BIT_PER_BYTE] &= ~(1UL << ((nr) % BIT_PER_BYTE)); + } + + static void SetBit(int nr, uint8_t addr[]) + { + addr[nr / BIT_PER_BYTE] |= (1UL << ((nr) % BIT_PER_BYTE)); + } + + static int FindNextBit(const uint8_t addr[], int maxSlots, int start) + { + while (start < maxSlots) { + if (BitOps::TestBit(start, addr)) { + return start; + } + start++; + } + return maxSlots; + } + + static int FindNextZeroBit(const uint8_t addr[], int maxSlots, int start) + { + while (start < maxSlots) { + if (!BitOps::TestBit(start, addr)) { + return start; + } + start++; + } + return maxSlots; + } +}; + +} // namespace FileManagement +} // namespace OHOS + +#endif // META_FILE_H diff --git a/utils/dentry/src/file_utils.cpp b/utils/dentry/src/file_utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8966e9c051598d08a10c36e74434f18333757a3c --- /dev/null +++ b/utils/dentry/src/file_utils.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "file_utils.h" + +#include +#include + +#include "dfs_error.h" +#include "utils_log.h" + +namespace OHOS { +namespace FileManagement { + +int32_t FileUtils::Stat(int fd, struct stat &s) +{ + if (fd < 0) { + LOGE("invalid fd %{public}d", fd); + return EBADF; + } + + if (fstat(fd, &s) != 0) { + LOGE("errno %{public}d, fd=%{public}d", errno, fd); + return errno; + } + + return E_OK; +} + +int32_t FileUtils::TruncateFile(int fd, size_t size) +{ + if (fd < 0) { + LOGE("invalid fd %{public}d", fd); + return EBADF; + } + + int ret = ftruncate(fd, size); + if (ret < 0) { + LOGE("errno %{public}d, fd=%{public}d", errno, fd); + return errno; + } + + return E_OK; +} + +int64_t FileUtils::ReadFile(int fd, off_t offset, size_t size, void *data) +{ + if ((fd < 0) || (offset < 0) || (size < 0) || (data == nullptr)) { + LOGE("invalid params, fd %{public}d, offset %{public}d, size %{public}zu, or buf is null", fd, + static_cast(offset), size); + return -1; + } + + size_t readLen = 0; + lseek(fd, offset, 0); + while (readLen < size) { + ssize_t ret = read(fd, data, size - readLen); + if (ret < 0) { + LOGE("errno %{public}d, fd=%{public}d", errno, fd); + return ret; + } else if (ret == 0) { + break; + } + readLen += ret; + } + + return readLen; +} + +int64_t FileUtils::WriteFile(int fd, const void *data, off_t offset, size_t size) +{ + if ((fd < 0) || (offset < 0) || (size < 0) || (data == nullptr)) { + LOGE("invalid params, fd %{public}d, offset %{public}d, size %{public}zu, or buf is null", fd, + static_cast(offset), size); + return -1; + } + + size_t writeLen = 0; + lseek(fd, offset, 0); + while (writeLen < size) { + ssize_t ret = write(fd, data, size - writeLen); + if ((ret < 0) || (ret == 0)) { + LOGE("errno %{public}d, fd=%{public}d", errno, fd); + return ret; + } + writeLen += ret; + } + + return writeLen; +} +} // namespace FileManagement +} // namespace OHOS diff --git a/utils/dentry/src/meta_file.cpp b/utils/dentry/src/meta_file.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5516319343f0636a779399ec7b0308cf39094265 --- /dev/null +++ b/utils/dentry/src/meta_file.cpp @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "meta_file.h" + +#include +#include +#include + +#include "dfs_error.h" +#include "directory_ex.h" +#include "file_utils.h" +#include "securec.h" +#include "string_ex.h" +#include "sys/xattr.h" +#include "utils_log.h" + +namespace OHOS { +namespace FileManagement { +constexpr uint32_t DENTRYGROUP_SIZE = 4096; +constexpr uint32_t DENTRY_NAME_LEN = 8; +constexpr uint32_t DENTRY_RESERVED_LENGTH = 3; +constexpr uint32_t DENTRY_PER_GROUP = 80; +constexpr uint32_t DENTRY_BITMAP_LENGTH = 10; +constexpr uint32_t DENTRY_GROUP_RESERVED = 5; +constexpr uint32_t DENTRYGROUP_HEADER = 4096; +constexpr uint32_t MAX_BUCKET_LEVEL = 63; +constexpr uint32_t BUCKET_BLOCKS = 2; +constexpr uint32_t BITS_PER_BYTE = 8; +constexpr uint32_t HMDFS_SLOT_LEN_BITS = 3; +constexpr uint64_t DELTA = 0x9E3779B9; /* Hashing code copied from f2fs */ +constexpr uint64_t HMDFS_HASH_COL_BIT = (0x1ULL) << 63; + +#pragma pack(push, 1) +struct HmdfsDentry { + uint32_t hash; + uint16_t mode; + uint16_t namelen; + uint64_t size; + /* modification time */ + uint64_t mtime; + /* modification time in nano scale */ + uint32_t mtimeNsec; + uint64_t ino; + uint32_t flag; + /* reserved bytes for long term extend, total 43 bytes */ + uint8_t reserved[DENTRY_RESERVED_LENGTH]; +}; + +struct HmdfsDentryGroup { + uint8_t dentryVersion; + uint8_t bitmap[DENTRY_BITMAP_LENGTH]; + struct HmdfsDentry nsl[DENTRY_PER_GROUP]; + uint8_t fileName[DENTRY_PER_GROUP][DENTRY_NAME_LEN]; + uint8_t reserved[DENTRY_GROUP_RESERVED]; +}; + +struct HmdfsDcacheHeader { + uint64_t dcacheCrtime; + uint64_t dcacheCrtimeNsec; + + uint64_t dentryCtime; + uint64_t dentryCtimeNsec; + + uint64_t num; + uint8_t case_sensitive; +}; +#pragma pack(pop) + +static uint64_t PathHash(const std::string &path, bool caseSense) +{ + uint64_t res = 0; + const char *kp = path.c_str(); + char c; + + while (*kp) { + c = *kp; + if (!caseSense) + c = tolower(c); + res = (res << 5) - res + (uint64_t)(c); + kp++; + } + return res; +} + +static std::string GetDentryfileHash(const std::string &path, bool caseSense) +{ + std::string fileHash = DexToHexString(PathHash(path, caseSense), false); + return fileHash; +} + +static std::string GetDentryfileByPath(uint32_t userId, const std::string &path, bool caseSense = false) +{ + std::string cacheDir = "/data/service/el2/" + std::to_string(userId) + "/hmdfs/cache/cloud_cache/"; + std::string dentryFileHash = GetDentryfileHash(path, caseSense); + + return cacheDir + dentryFileHash; +} + +MetaFile::MetaFile(uint32_t userId, const std::string &parentPath) +{ + std::string cacheFile = GetDentryfileByPath(userId, parentPath); + fd_ = UniqueFd{open(cacheFile.c_str(), O_RDWR | O_CREAT)}; + + struct stat statBuf {}; + int ret = fstat(fd_, &statBuf); + LOGI("fd=%{public}d, size=%{public}u, parentPath=%s", fd_.Get(), (uint32_t)statBuf.st_size, cacheFile.c_str()); + + ret = fsetxattr(fd_, "user.hmdfs_cache", parentPath.c_str(), parentPath.size(), 0); + if (ret != 0) { + LOGE("setxattr failed, errno %{public}d, parentPath %s", errno, parentPath.c_str()); + } +} + +static bool IsDotDotdot(const std::string &name) +{ + return name == "." || name == ".."; +} + +static void Str2HashBuf(const char *msg, size_t len, uint32_t *buf, int num) +{ + uint32_t pad = static_cast(len) | (static_cast(len) << 8); + pad |= pad << 16; + + uint32_t val = pad; + if (len > static_cast(num * 4)) { + len = static_cast(num * 4); + } + for (uint32_t i = 0; i < len; i++) { + if ((i % 4) == 0) { + val = pad; + } + uint8_t c = tolower(msg[i]); + val = c + (val << 8); + if ((i % 4) == 3) { + *buf++ = val; + val = pad; + num--; + } + } + if (--num >= 0) { + *buf++ = val; + } + while (--num >= 0) { + *buf++ = pad; + } +} + +static void TeaTransform(uint32_t buf[4], uint32_t const in[]) +{ + int n = 16; + uint32_t sum = 0; + uint32_t a = in[0]; + uint32_t b = in[1]; + uint32_t c = in[2]; + uint32_t d = in[3]; + uint32_t b0 = buf[0]; + uint32_t b1 = buf[1]; + + do { + sum += DELTA; + b0 += ((b1 << 4) + a) ^ (b1 + sum) ^ ((b1 >> 5) + b); + b1 += ((b0 << 4) + c) ^ (b0 + sum) ^ ((b0 >> 5) + d); + } while (--n); + + buf[0] += b0; + buf[1] += b1; +} + +static uint32_t DentryHash(const std::string &name) +{ + if (IsDotDotdot(name)) { + return 0; + } + + const int inLen = 8; + const int bufLen = 4; + uint32_t in[inLen], buf[bufLen]; + uint32_t len = name.length(); + const char *p = name.c_str(); + + buf[0] = 0x67452301; + buf[1] = 0xefcdab89; + buf[2] = 0x98badcfe; + buf[3] = 0x10325476; + + do { + Str2HashBuf(p, len, in, 4); + TeaTransform(buf, in); + p += 16; + } while (len-- <= 16); + uint32_t hash = buf[0]; + uint32_t hmdfsHash = hash & ~HMDFS_HASH_COL_BIT; + + return hmdfsHash; +} + +static inline int GetDentrySlots(size_t nameLen) +{ + return (nameLen + BITS_PER_BYTE - 1) >> HMDFS_SLOT_LEN_BITS; +} + +static inline off_t GetDentryGroupPos(size_t bidx) +{ + return (off_t)bidx * DENTRYGROUP_SIZE + DENTRYGROUP_HEADER; +} + +static inline uint64_t GetDentryGroupCnt(uint64_t size) +{ + return (size >= DENTRYGROUP_HEADER) ? ((size - DENTRYGROUP_HEADER) / DENTRYGROUP_SIZE) : 0; +} + +static uint32_t GetOverallBucket(uint32_t level) +{ + uint32_t buckets = 0; + + if (level >= MAX_BUCKET_LEVEL) { + LOGI("level = %{public}d overflow", level); + return buckets; + } + buckets = (1U << (level + 1)) - 1; + + return buckets; +} + +static size_t GetDcacheFileSize(uint32_t level) +{ + size_t buckets = GetOverallBucket(level); + + return buckets * DENTRYGROUP_SIZE * BUCKET_BLOCKS + DENTRYGROUP_HEADER; +} + +static uint32_t GetBucketaddr(uint32_t level, uint32_t buckoffset) +{ + if (level >= MAX_BUCKET_LEVEL) { + LOGI("level = %{public}d overflow", level); + return 0; + } + uint32_t curLevelMaxBucks = (1U << level); + if (buckoffset >= curLevelMaxBucks) { + LOGI("buckoffset %{public}d overflow, level %{public}d has %{public}d buckets max", buckoffset, level, + curLevelMaxBucks); + return 0; + } + + return curLevelMaxBucks + buckoffset - 1; +} + +static uint32_t GetBucketByLevel(uint32_t level) +{ + if (level >= MAX_BUCKET_LEVEL) { + LOGI("level = %{public}d overflow", level); + return 0; + } + + uint32_t buckets = (1U << level); + return buckets; +} + +static int RoomForFilename(const uint8_t bitmap[], int slots, int maxSlots) +{ + int bitStart = 0; + + while (1) { + int zeroStart = BitOps::FindNextZeroBit(bitmap, maxSlots, bitStart); + if (zeroStart >= maxSlots) { + return maxSlots; + } + + int zeroEnd = BitOps::FindNextBit(bitmap, maxSlots, zeroStart); + if (zeroEnd - zeroStart >= slots) { + return zeroStart; + } + + bitStart = zeroEnd + 1; + if (zeroEnd + 1 >= maxSlots) { + return maxSlots; + } + } + return 0; +} + +static void UpdateDentry(struct HmdfsDentryGroup &d, const MetaBase &base, uint32_t nameHash, uint32_t bitPos) +{ + struct HmdfsDentry *de; + const std::string name = base.name; + int slots = GetDentrySlots(name.length()); + + de = &d.nsl[bitPos]; + de->hash = nameHash; + de->namelen = (uint16_t)name.length(); + if (memcpy_s(d.fileName[bitPos], slots * DENTRY_NAME_LEN, name.c_str(), name.length())) { + LOGE("memcpy_s failed, dstLen = %{public}d, srcLen = %{public}zu", slots * DENTRY_NAME_LEN, name.length()); + } + de->mtime = base.mtime; + de->mtimeNsec = 0; + de->size = base.size; + de->ino = base.ino; + de->flag = 0; + de->mode = (uint16_t)base.mode; + + for (int i = 0; i < slots; i++) { + BitOps::SetBit(bitPos + i, d.bitmap); + if (i) { + (de + i)->namelen = 0; + } + } +} + +int32_t MetaFile::DoCreate(const MetaBase &base) +{ + off_t pos = 0; + uint32_t level = 0; + uint32_t bitPos, namehash; + unsigned long bidx, endBlock; + HmdfsDentryGroup dentryBlk = {0}; + + namehash = DentryHash(base.name); + + while (1) { + if (level == MAX_BUCKET_LEVEL) { + return ENOENT; + } + + bidx = BUCKET_BLOCKS * GetBucketaddr(level, namehash % GetBucketByLevel(level)); + endBlock = bidx + BUCKET_BLOCKS; + + struct stat fileStat; + int err = FileUtils::Stat(fd_, fileStat); + if (err != E_OK) { + return EINVAL; + } + + if ((endBlock > GetDentryGroupCnt(fileStat.st_size)) && FileUtils::TruncateFile(fd_, GetDcacheFileSize(level))) { + return ENOENT; + } + + for (; bidx < endBlock; bidx++) { + pos = GetDentryGroupPos(bidx); + int size = FileUtils::ReadFile(fd_, pos, DENTRYGROUP_SIZE, &dentryBlk); + if (size != DENTRYGROUP_SIZE) { + return ENOENT; + } + + bitPos = RoomForFilename(dentryBlk.bitmap, GetDentrySlots(base.name.length()), DENTRY_PER_GROUP); + if (bitPos < DENTRY_PER_GROUP) { + goto ADD; + } + } + ++level; + } + +ADD: + pos = GetDentryGroupPos(bidx); + UpdateDentry(dentryBlk, base, namehash, bitPos); + int size = FileUtils::WriteFile(fd_, &dentryBlk, pos, DENTRYGROUP_SIZE); + if (size != DENTRYGROUP_SIZE) { + LOGI("WriteFile failed, size %{public}d != %{public}d", size, DENTRYGROUP_SIZE); + return EINVAL; + } + return E_OK; +} +} // namespace FileManagement +} // namespace OHOS