diff --git a/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h b/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h index af35f7a4a82d86881f45867e043560fa043aad47..97bdd75a26a64516262c23b5602fd88f63e1dd44 100644 --- a/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h +++ b/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h @@ -16,6 +16,10 @@ #ifndef CLOUD_DB_CONSTANT_H #define CLOUD_DB_CONSTANT_H +#ifndef MAX_UPLOAD_COUNT +#define MAX_UPLOAD_COUNT INT32_MAX +#endif // MAX_UPLOAD_COUNT + #include "cloud/cloud_store_types.h" #include diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_extend_executor.cpp b/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_extend_executor.cpp index 170e611a002959c98afecfefaea2a370dc87da53..388d7113bb43038dee0588278e8b2275ffa9e7d1 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_extend_executor.cpp +++ b/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_extend_executor.cpp @@ -255,15 +255,9 @@ bool SQLiteSingleVerRelationalStorageExecutor::IsGetCloudDataContinue(uint32_t c if (curNum == 0) { return true; } -#ifdef MAX_UPLOAD_COUNT if (curSize < maxSize && curNum < MAX_UPLOAD_COUNT) { return true; } -#else - if (curSize < maxSize) { - return true; - } -#endif return false; } } // namespace DistributedDB diff --git a/frameworks/libs/distributeddb/test/BUILD.gn b/frameworks/libs/distributeddb/test/BUILD.gn index 55b4ce2a2e401d28d53f6bd2df80aa98361aa776..fedfd9679bc900867cb95723349cf83a12e08b23 100644 --- a/frameworks/libs/distributeddb/test/BUILD.gn +++ b/frameworks/libs/distributeddb/test/BUILD.gn @@ -72,6 +72,10 @@ config("module_private_config") { ] } +config("ut_extra_config") { + defines = [ "MAX_UPLOAD_COUNT=30" ] +} + ############################################################################### ohos_source_set("src_file") { testonly = true @@ -395,6 +399,38 @@ distributeddb_unittest("DistributedDBCloudInterfacesRelationalSyncTest") { sources = [ "unittest/common/storage/cloud/distributeddb_cloud_interfaces_relational_sync_test.cpp" ] } +ohos_unittest("DistributedDBCloudInterfacesRelationalSyncCountLimitTest") { + module_out_path = module_output_path + configs = [ ":module_private_config" ] + configs += [ ":ut_extra_config" ] + configs += [ "//third_party/jsoncpp:jsoncpp_config" ] + if (!defined(deps)) { + deps = [] + } + deps += [ + ":src_file", + "//third_party/googletest:gmock_main", + "//third_party/googletest:gtest_main", + "//third_party/sqlite:sqlite", + "//third_party/zlib:libz", + ] + deps += [ + "//third_party/jsoncpp:jsoncpp", + "//third_party/openssl:libcrypto_shared", + ] + if (!defined(external_deps)) { + external_deps = [] + } + ldflags = [ "-Wl,--exclude-libs,ALL" ] + external_deps = [ + "c_utils:utils", + "hilog:libhilog", + "hisysevent:libhisysevent", + "hitrace:hitrace_meter", + ] + sources = [ "unittest/common/storage/cloud/distributeddb_cloud_interfaces_relational_sync_count_limit_test.cpp" ] +} + distributeddb_unittest("DistributedDBCloudSyncerProgressManagerTest") { sources = [ "unittest/common/syncer/cloud/distributeddb_cloud_syncer_progress_manager_test.cpp" ] } @@ -863,6 +899,7 @@ group("unittest") { ":DistributedDBCloudAssetCompareTest", ":DistributedDBCloudDBProxyTest", ":DistributedDBCloudInterfacesRelationalExtTest", + ":DistributedDBCloudInterfacesRelationalSyncCountLimitTest", ":DistributedDBCloudInterfacesRelationalSyncTest", ":DistributedDBCloudSaveCloudDataTest", ":DistributedDBCloudSchemaMgrTest", diff --git a/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_interfaces_relational_sync_count_limit_test.cpp b/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_interfaces_relational_sync_count_limit_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9570d74744ea6315d1871edb484186f2f215e3a4 --- /dev/null +++ b/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_interfaces_relational_sync_count_limit_test.cpp @@ -0,0 +1,356 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include +#include +#include "cloud/cloud_storage_utils.h" +#include "cloud_db_constant.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "process_system_api_adapter_impl.h" +#include "relational_store_instance.h" +#include "relational_store_manager.h" +#include "runtime_config.h" +#include "sqlite_relational_store.h" +#include "sqlite_relational_utils.h" +#include "store_observer.h" +#include "time_helper.h" +#include "virtual_asset_loader.h" +#include "virtual_cloud_data_translate.h" +#include "virtual_cloud_db.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { + string g_testDir; + string g_storePath; + string g_storeID = "Relational_Store_SYNC"; + const string g_tableName1 = "user1"; + const string DB_SUFFIX = ".db"; + const string CLOUD_DATA = "cloudData"; + const string CLOUD_ASSET = "cloudAsset"; + const string CLOUD_HASH = "cloudHash"; + const string LOCAL_DATA = "localData"; + const string LOCAL_ASSET = "localAsset"; + const string LOCAL_HASH = "localHash"; + const string DEVICE_CLOUD = "cloud_dev"; + int g_syncIndex = 0; + std::mutex g_processMutex; + Assets g_cloudAssets; + Assets g_localAssets; + DistributedDB::RelationalStoreManager g_mgr(APP_ID, USER_ID); + RelationalStoreObserverUnitTest *g_observer = nullptr; + RelationalStoreDelegate *g_delegate = nullptr; + std::shared_ptr g_virtualCloudDb; + SyncProcess g_syncProcess; + using CloudSyncStatusCallback = std::function &onProcess)>; + std::condition_variable g_processCondition; + const std::vector g_tables = {g_tableName1}; + const int64_t g_syncWaitTime = 10; + + struct TableOptions { + int startId; + int rowCount; + bool needAsset; + }; + + const std::string INTEGER_PRIMARY_KEY_TABLE_SQL = + "CREATE TABLE IF NOT EXISTS " + g_tableName1 + "(" \ + "id INTEGER PRIMARY KEY," \ + "name TEXT ," \ + "assert BLOB, " \ + "asserts BLOB);"; + + const std::vector g_cloudFiled1 = { + {"id", TYPE_INDEX, true}, {"name", TYPE_INDEX}, + {"assert", TYPE_INDEX}, {"asserts", TYPE_INDEX} + }; + + void GetCloudDbSchema(DataBaseSchema &dataBaseSchema) + { + TableSchema tableSchema1 = { + .name = g_tableName1, + .fields = g_cloudFiled1 + }; + dataBaseSchema.tables.push_back(tableSchema1); + } + + Asset GenAsset(std::string name, std::string hash) + { + Asset asset; + asset.name = name; + asset.hash = hash; + return asset; + } + + VBucket GenDatum(int64_t id, std::string name, Type asset, Type assets) + { + VBucket datum; + int index = 0; + datum.insert_or_assign(g_cloudFiled1[index++].colName, id); + datum.insert_or_assign(g_cloudFiled1[index++].colName, name); + datum.insert_or_assign(g_cloudFiled1[index++].colName, asset); + datum.insert_or_assign(g_cloudFiled1[index].colName, assets); + return datum; + } + + void InitAsset() + { + for (int i = 0; i < 4; ++i) { // 4 is the num of Asset + g_cloudAssets.push_back(GenAsset(CLOUD_ASSET + std::to_string(i), CLOUD_HASH + std::to_string(i))); + g_localAssets.push_back(GenAsset(LOCAL_ASSET + std::to_string(i), LOCAL_HASH + std::to_string(i))); + } + } + + void CreateUserDBAndTable(sqlite3 *&db) + { + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, INTEGER_PRIMARY_KEY_TABLE_SQL), SQLITE_OK); + } + + void InsertCloudTableRecord(TableOptions &options) + { + std::vector record1; + std::vector extend1; + Timestamp now = TimeHelper::GetSysCurrentTime(); + for (int64_t i = options.startId; i < options.startId + options.rowCount; ++i) { + VBucket data; + if (options.needAsset) { + data = GenDatum(i, CLOUD_DATA + std::to_string(i), g_cloudAssets[0], g_cloudAssets); + } else { + data = GenDatum(i, CLOUD_DATA + std::to_string(i), Nil(), Nil()); + } + record1.push_back(data); + VBucket log; + log.insert_or_assign(CloudDbConstant::CREATE_FIELD, (int64_t)now / CloudDbConstant::TEN_THOUSAND + i); + log.insert_or_assign(CloudDbConstant::MODIFY_FIELD, (int64_t)now / CloudDbConstant::TEN_THOUSAND + i); + log.insert_or_assign(CloudDbConstant::DELETE_FIELD, false); + extend1.push_back(log); + } + ASSERT_EQ(g_virtualCloudDb->BatchInsert(g_tableName1, std::move(record1), extend1), DBStatus::OK); + std::this_thread::sleep_for(std::chrono::milliseconds(options.rowCount)); + } + + void InsertLocalTableRecord(sqlite3 *&db, TableOptions &options) + { + int errCode; + std::vector assetBlob; + std::vector assetsBlob; + RuntimeContext::GetInstance()->AssetToBlob(g_localAssets[0], assetBlob); + RuntimeContext::GetInstance()->AssetsToBlob(g_localAssets, assetsBlob); + for (int64_t i = options.startId; i < options.startId + options.rowCount; ++i) { + string sql = "INSERT OR REPLACE INTO " + g_tableName1 + + " (id, name, assert, asserts) VALUES ('" + std::to_string(i) + "', '" + LOCAL_DATA + + std::to_string(i) + "', ? , ?);"; + sqlite3_stmt *stmt = nullptr; + ASSERT_EQ(SQLiteUtils::GetStatement(db, sql, stmt), E_OK); + if (options.needAsset) { + ASSERT_EQ(SQLiteUtils::BindBlobToStatement(stmt, 1, assetBlob, false), E_OK); + ASSERT_EQ(SQLiteUtils::BindBlobToStatement(stmt, 2, assetBlob, false), E_OK); // 2 is assets index + } else { + ASSERT_EQ(sqlite3_bind_null(stmt, 1), E_OK); + ASSERT_EQ(sqlite3_bind_null(stmt, 2), E_OK); // 2 is assets index + } + EXPECT_EQ(SQLiteUtils::StepWithRetry(stmt), SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)); + SQLiteUtils::ResetStatement(stmt, true, errCode); + } + } + + void InitProcessForTest1(const uint32_t &cloudCount, const uint32_t &localCount, + std::vector &expectProcess) + { + expectProcess.clear(); + std::vector infos; + uint32_t index = 1; + infos.push_back(TableProcessInfo{ + PROCESSING, {index, cloudCount, cloudCount, 0}, {0, 0, 0, 0} + }); + + infos.push_back(TableProcessInfo{ + PROCESSING, {index, cloudCount, cloudCount, 0}, + {index, cloudCount, MAX_UPLOAD_COUNT, 0} + }); + + infos.push_back(TableProcessInfo{ + FINISHED, {index, cloudCount, cloudCount, 0}, + {index + 1, cloudCount, cloudCount, 0} + }); + + for (size_t i = 0; i <= infos.size(); ++i) { + SyncProcess syncProcess; + syncProcess.errCode = OK; + syncProcess.process = i == infos.size() ? FINISHED : PROCESSING; + syncProcess.tableProcess.insert_or_assign(g_tables[0], std::move(infos[i])); + expectProcess.push_back(syncProcess); + } + } + + void GetCallback(SyncProcess &syncProcess, CloudSyncStatusCallback &callback, + std::vector &expectProcess) + { + g_syncIndex = 0; + callback = [&syncProcess, &expectProcess](const std::map &process) { + LOGI("devices size = %d", process.size()); + ASSERT_EQ(process.size(), 1u); + syncProcess = std::move(process.begin()->second); + ASSERT_EQ(process.begin()->first, DEVICE_CLOUD); + ASSERT_NE(syncProcess.tableProcess.empty(), true); + LOGI("current sync process status:%d, db status:%d ", syncProcess.process, syncProcess.errCode); + std::for_each(g_tables.begin(), g_tables.end(), [&](const auto &item) { + auto table1 = syncProcess.tableProcess.find(item); + if (table1 != syncProcess.tableProcess.end()) { + LOGI("table[%s], table process status:%d, [downloadInfo](batchIndex:%u, total:%u, successCount:%u, " + "failCount:%u) [uploadInfo](batchIndex:%u, total:%u, successCount:%u,failCount:%u", + item.c_str(), table1->second.process, table1->second.downLoadInfo.batchIndex, + table1->second.downLoadInfo.total, table1->second.downLoadInfo.successCount, + table1->second.downLoadInfo.failCount, table1->second.upLoadInfo.batchIndex, + table1->second.upLoadInfo.total, table1->second.upLoadInfo.successCount, + table1->second.upLoadInfo.failCount); + } + }); + if (expectProcess.empty()) { + if (syncProcess.process == FINISHED) { + g_processCondition.notify_one(); + } + return; + } + ASSERT_LE(static_cast(g_syncIndex), expectProcess.size()); + for (size_t i = 0; i < g_tables.size(); ++i) { + SyncProcess head = expectProcess[g_syncIndex]; + for (auto &expect : head.tableProcess) { + auto real = syncProcess.tableProcess.find(expect.first); + ASSERT_NE(real, syncProcess.tableProcess.end()); + EXPECT_EQ(expect.second.process, real->second.process); + EXPECT_EQ(expect.second.downLoadInfo.batchIndex, real->second.downLoadInfo.batchIndex); + EXPECT_EQ(expect.second.downLoadInfo.total, real->second.downLoadInfo.total); + EXPECT_EQ(expect.second.downLoadInfo.successCount, real->second.downLoadInfo.successCount); + EXPECT_EQ(expect.second.downLoadInfo.failCount, real->second.downLoadInfo.failCount); + EXPECT_EQ(expect.second.upLoadInfo.batchIndex, real->second.upLoadInfo.batchIndex); + EXPECT_EQ(expect.second.upLoadInfo.total, real->second.upLoadInfo.total); + EXPECT_EQ(expect.second.upLoadInfo.successCount, real->second.upLoadInfo.successCount); + EXPECT_EQ(expect.second.upLoadInfo.failCount, real->second.upLoadInfo.failCount); + } + } + g_syncIndex++; + if (syncProcess.process == FINISHED) { + g_processCondition.notify_one(); + } + }; + } + + void WaitForSyncFinish(SyncProcess &syncProcess, const int64_t &waitTime) + { + std::unique_lock lock(g_processMutex); + bool result = g_processCondition.wait_for(lock, std::chrono::seconds(waitTime), [&syncProcess]() { + return syncProcess.process == FINISHED; + }); + ASSERT_EQ(result, true); + LOGD("-------------------sync end--------------"); + } + + void CloseDb() + { + delete g_observer; + g_virtualCloudDb = nullptr; + if (g_delegate != nullptr) { + EXPECT_EQ(g_mgr.CloseStore(g_delegate), DBStatus::OK); + g_delegate = nullptr; + } + } + + class DistributedDBCloudInterfacesRelationalSyncCountLimitTest : public testing::Test { + public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); + protected: + sqlite3 *db = nullptr; + }; + + + void DistributedDBCloudInterfacesRelationalSyncCountLimitTest::SetUpTestCase(void) + { + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + g_storePath = g_testDir + "/" + g_storeID + DB_SUFFIX; + LOGI("The test db is:%s", g_testDir.c_str()); + RuntimeConfig::SetCloudTranslate(std::make_shared()); + InitAsset(); + } + + void DistributedDBCloudInterfacesRelationalSyncCountLimitTest::TearDownTestCase(void) + {} + + void DistributedDBCloudInterfacesRelationalSyncCountLimitTest::SetUp(void) + { + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error."); + } + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + LOGD("Test dir is %s", g_testDir.c_str()); + db = RelationalTestUtils::CreateDataBase(g_storePath); + ASSERT_NE(db, nullptr); + CreateUserDBAndTable(db); + g_observer = new (std::nothrow) RelationalStoreObserverUnitTest(); + ASSERT_NE(g_observer, nullptr); + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option { .observer = g_observer }, + g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName1, CLOUD_COOPERATION), DBStatus::OK); + g_virtualCloudDb = make_shared(); + g_syncProcess = {}; + ASSERT_EQ(g_delegate->SetCloudDB(g_virtualCloudDb), DBStatus::OK); + ASSERT_EQ(g_delegate->SetIAssetLoader(make_shared()), DBStatus::OK); + DataBaseSchema dataBaseSchema; + GetCloudDbSchema(dataBaseSchema); + ASSERT_EQ(g_delegate->SetCloudDbSchema(dataBaseSchema), DBStatus::OK); + } + + void DistributedDBCloudInterfacesRelationalSyncCountLimitTest::TearDown(void) + { + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error."); + } + } + +/** + * @tc.name: CloudSyncLimitTest001 + * @tc.desc: Test macro UPLOAD_COUNT_LIMIT + * @tc.type: FUNC + * @tc.require: + * @tc.author: bty + */ +HWTEST_F(DistributedDBCloudInterfacesRelationalSyncCountLimitTest, CloudSyncLimitTest001, TestSize.Level0) +{ + TableOptions options = { + .startId = 0, .rowCount = 60, .needAsset = true + }; + InsertCloudTableRecord(options); + InsertLocalTableRecord(db, options); + Query query = Query::Select().FromTable(g_tables); + std::vector expectProcess; + InitProcessForTest1(options.rowCount, options.rowCount, expectProcess); + CloudSyncStatusCallback callback; + GetCallback(g_syncProcess, callback, expectProcess); + ASSERT_EQ(g_delegate->Sync({DEVICE_CLOUD}, SYNC_MODE_CLOUD_MERGE, query, callback, g_syncWaitTime), DBStatus::OK); + WaitForSyncFinish(g_syncProcess, g_syncWaitTime); + CloseDb(); +} +} +#endif // RELATIONAL_STORE \ No newline at end of file