diff --git a/src/gausskernel/storage/mot/core/src/infra/config/config_section.cpp b/src/gausskernel/storage/mot/core/src/infra/config/config_section.cpp index 3ed6d59e274d9e79360ade024ca97899777e969b..2e396494aa5bdb39821de6426b8f0e3f7046f93b 100644 --- a/src/gausskernel/storage/mot/core/src/infra/config/config_section.cpp +++ b/src/gausskernel/storage/mot/core/src/infra/config/config_section.cpp @@ -130,7 +130,7 @@ bool ConfigSection::GetConfigValueNames(mot_string_list& valueNames) const return true; } -bool ConfigSection::AddConfigItem(ConfigItem* configItem) +bool ConfigSection::AddConfigItem(ConfigItem* configItem, bool replaceIfExists /* = false */) { bool result = false; switch (configItem->GetClass()) { @@ -139,7 +139,7 @@ bool ConfigSection::AddConfigItem(ConfigItem* configItem) break; case ConfigItemClass::CONFIG_ITEM_VALUE: - result = AddConfigValue(static_cast(configItem)); + result = AddConfigValue(static_cast(configItem), replaceIfExists); break; case ConfigItemClass::CONFIG_ITEM_ARRAY: @@ -271,15 +271,22 @@ bool ConfigSection::AddConfigSection(ConfigSection* configSection) return (pairis.second == INSERT_SUCCESS); } -bool ConfigSection::AddConfigValue(ConfigValue* configValue) +bool ConfigSection::AddConfigValue(ConfigValue* configValue, bool replaceIfExists) { ConfigValueMap::pairis pairis = m_valueMap.insert(ConfigValueMap::value_type(configValue->GetName(), configValue)); if (pairis.second == INSERT_EXISTS) { - MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, - "Load Configuration", - "Cannot add configuration value %s to section %s: value already exists", - configValue->GetName(), - GetName()); + if (replaceIfExists) { + ConfigValue* prevValue = pairis.first->second; + pairis.first->second = configValue; + delete prevValue; + pairis.second = INSERT_SUCCESS; + } else { + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, + "Load Configuration", + "Cannot add configuration value %s to section %s: value already exists", + configValue->GetName(), + GetName()); + } } else if (pairis.second == INSERT_FAILED) { MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, "Load Configuration", diff --git a/src/gausskernel/storage/mot/core/src/infra/config/config_section.h b/src/gausskernel/storage/mot/core/src/infra/config/config_section.h index 05c95ceacb88b46889bcfe2d2c75e9116d9f1f5d..9ad5252d9896c760bbf6aa22faa9276c027d1ddd 100644 --- a/src/gausskernel/storage/mot/core/src/infra/config/config_section.h +++ b/src/gausskernel/storage/mot/core/src/infra/config/config_section.h @@ -86,11 +86,13 @@ public: /** * @brief Builds the section by adding a sub-item. - * @param config_item The direct configuration item to add as a child of this section. It could be - * another section or a terminal value. + * @param config_item The direct configuration item to add as a child of this section. It could be another section + * or a terminal value. + * @param[opt] replaceIfExists Specifies whether to replace an existing item if it already exists by the given name. + * This is relevant only for configuration values, not for configuration sections or arrays. * @return True if the item added or false if a configuration item by that name already exists. */ - bool AddConfigItem(ConfigItem* config_item); + bool AddConfigItem(ConfigItem* configItem, bool replaceIfExists = false); /** * @brief Merges the given section into this section. @@ -225,7 +227,7 @@ public: * @return The typed sub-value, or null pointer if not found. */ template - inline TypedConfigValue* MmodifyConfigValue(const char* name) + inline TypedConfigValue* ModifyConfigValue(const char* name) { TypedConfigValue* result = nullptr; ConfigValueMap::iterator itr = m_valueMap.find(name); @@ -286,9 +288,10 @@ private: /** * @brief Builds the section by adding a sub-value. * @param configValue The direct configuration sub-value to add as a child of this section. + * @param replaceIfExists Specifies whether to replace an existing item if it already exists by the given name. * @return True if the item added or false if a configuration item by that name already exists. */ - bool AddConfigValue(ConfigValue* configValue); + bool AddConfigValue(ConfigValue* configValue, bool replaceIfExists); /** * @brief Builds the section by adding a sub-array. diff --git a/src/gausskernel/storage/mot/core/src/infra/config/config_tree.h b/src/gausskernel/storage/mot/core/src/infra/config/config_tree.h index 8401e36eaf368efdd6f3978346e91951762dfd0e..30408b4f0d9bbb289c64497cbf167359615e50f7 100644 --- a/src/gausskernel/storage/mot/core/src/infra/config/config_tree.h +++ b/src/gausskernel/storage/mot/core/src/infra/config/config_tree.h @@ -229,7 +229,7 @@ public: * @param defaultValue * @return */ - inline const char* getStringConfigValue(const char* fullPathName, const char* defaultValue) const + inline const char* GetStringConfigValue(const char* fullPathName, const char* defaultValue) const { const char* result = defaultValue; const StringConfigValue* configValue = GetQualifiedConfigItem(fullPathName); diff --git a/src/gausskernel/storage/mot/core/src/infra/config/config_value.h b/src/gausskernel/storage/mot/core/src/infra/config/config_value.h index 38a83bc2d274ffe3a7d2cf2327df6bff48a163b0..7bb8b5c6549c8de103d67d99b9ae1d8789c7ae3e 100644 --- a/src/gausskernel/storage/mot/core/src/infra/config/config_value.h +++ b/src/gausskernel/storage/mot/core/src/infra/config/config_value.h @@ -37,7 +37,9 @@ protected: * @param valueType The value type of this configuration value. */ explicit ConfigValue(ConfigValueType valueType) - : ConfigItem(ConfigItemClass::CONFIG_ITEM_VALUE), m_valueType(valueType) + : ConfigItem(ConfigItemClass::CONFIG_ITEM_VALUE), + m_valueType(valueType), + m_isIntegral(IsConfigValueIntegral(valueType)) {} public: @@ -54,9 +56,18 @@ public: return m_valueType; } + /** @brief Queries whether the value type is integral. */ + inline bool IsIntegral() const + { + return m_isIntegral; + } + private: /** @var The configuration value type. */ ConfigValueType m_valueType; + + /** @var Specifies whether this is an integral value type. */ + bool m_isIntegral; }; // specialization diff --git a/src/gausskernel/storage/mot/core/src/infra/config/config_value_type.cpp b/src/gausskernel/storage/mot/core/src/infra/config/config_value_type.cpp index ebe09b801117abf734e75ca00eef240979722f8d..5fc3dca1d3395027a4086e021ac008eb6fa3e79b 100644 --- a/src/gausskernel/storage/mot/core/src/infra/config/config_value_type.cpp +++ b/src/gausskernel/storage/mot/core/src/infra/config/config_value_type.cpp @@ -90,4 +90,16 @@ extern const char* ConfigValueTypeToString(ConfigValueType valueType) } return CONFIG_VALUE_UNDEFINED_STR; } + +extern bool IsConfigValueIntegral(ConfigValueType valueType) +{ + bool result = false; + if ((valueType == ConfigValueType::CONFIG_VALUE_INT64) || (valueType == ConfigValueType::CONFIG_VALUE_INT32) || + (valueType == ConfigValueType::CONFIG_VALUE_INT16) || (valueType == ConfigValueType::CONFIG_VALUE_INT8) || + (valueType == ConfigValueType::CONFIG_VALUE_UINT64) || (valueType == ConfigValueType::CONFIG_VALUE_UINT32) || + (valueType == ConfigValueType::CONFIG_VALUE_UINT16) || (valueType == ConfigValueType::CONFIG_VALUE_UINT8)) { + result = true; + } + return result; +} } // namespace MOT diff --git a/src/gausskernel/storage/mot/core/src/infra/config/config_value_type.h b/src/gausskernel/storage/mot/core/src/infra/config/config_value_type.h index 28c21cea33674690aa57e812c02c3451c753f552..f40c7c7e88ca9a78098d00cd42b46a21d4bf4643 100644 --- a/src/gausskernel/storage/mot/core/src/infra/config/config_value_type.h +++ b/src/gausskernel/storage/mot/core/src/infra/config/config_value_type.h @@ -63,7 +63,7 @@ enum class ConfigValueType : uint32_t { /** @var The configuration value is of type bool. */ CONFIG_VALUE_BOOL, - /** @var The configuration value is of type std::string. */ + /** @var The configuration value is of type MOT::mot_string. */ CONFIG_VALUE_STRING, /** @var The configuration value is undefined (parse error). */ @@ -83,6 +83,9 @@ extern ConfigValueType ConfigValueTypeFromString(const char* valueTypeStr); * @return The resulting configuration value type string. */ extern const char* ConfigValueTypeToString(ConfigValueType valueType); + +/** @brief Queries whether a value type is an integral type. */ +extern bool IsConfigValueIntegral(ConfigValueType valueType); } // namespace MOT #endif /* CONFIG_VALUE_TYPE_H */ diff --git a/src/gausskernel/storage/mot/core/src/infra/config/layered_config_tree.h b/src/gausskernel/storage/mot/core/src/infra/config/layered_config_tree.h index 97077aac0eaed537a3b3a9de13abcdf2648387cf..9b20cb75505dcc4a443f8286d2fad8e2b4f2ef35 100644 --- a/src/gausskernel/storage/mot/core/src/infra/config/layered_config_tree.h +++ b/src/gausskernel/storage/mot/core/src/infra/config/layered_config_tree.h @@ -92,13 +92,13 @@ public: * @param fullPathName The full path name of the configuration array. * @return The configuration array, or null pointer if none was found by this name. */ - inline const ConfigArray* getConfigArray(const char* fullPathName) const + inline const ConfigArray* GetConfigArray(const char* fullPathName) const { return GetQualifiedConfigItem(fullPathName); } /** - * @brief Retrieves a typed configuration value. + * @brief Retrieves a typed configuration value. For string types use @ref GetStringConfigValue(). * @param fullPathName The full path name of the configuration value. * @param defaultValue The default value to return if not found. * @return The configuration value, or null pointer if none was found by this name. @@ -106,14 +106,10 @@ public: template inline const T& GetConfigValue(const char* fullPathName, const T& defaultValue) const { - const TypedConfigValue* configValue = GetQualifiedConfigItem>(fullPathName); + const TypedConfigValue* configValue = (const TypedConfigValue*)GetQualifiedConfigValue(fullPathName); if (configValue != nullptr) { - if (configValue->GetConfigValueType() != ConfigValueTypeMapper::CONFIG_VALUE_TYPE) { - MOT_LOG_ERROR("Unexpected item value type: %s (%s instead of %s %d)", - fullPathName, - ConfigValueTypeToString(configValue->GetConfigValueType()), - ConfigValueTypeToString(ConfigValueTypeMapper::CONFIG_VALUE_TYPE), - (int)ConfigValueTypeMapper::CONFIG_VALUE_TYPE); + if (!configValue->IsValid()) { + MOT_LOG_ERROR("Configuration value of %s is invalid (OOM in string allocation?)", fullPathName); } else { return configValue->GetValue(); } @@ -131,9 +127,14 @@ public: inline const char* GetStringConfigValue(const char* fullPathName, const char* defaultValue) const { const char* result = defaultValue; - const StringConfigValue* configValue = GetQualifiedConfigItem(fullPathName); + const StringConfigValue* configValue = + (const StringConfigValue*)GetQualifiedConfigValue(fullPathName); if (configValue != nullptr) { - result = configValue->GetValue().c_str(); + if (!configValue->IsValid()) { + MOT_LOG_ERROR("Configuration value of %s is invalid (OOM in string allocation?)", fullPathName); + } else { + result = configValue->GetValue().c_str(); + } } return result; } @@ -149,9 +150,16 @@ public: inline T GetUserConfigValue(const char* fullPathName, const T& defaultValue) const { T result = defaultValue; - const StringConfigValue* configValue = GetQualifiedConfigItem(fullPathName); + const StringConfigValue* configValue = + (const StringConfigValue*)GetQualifiedConfigValue(fullPathName); if (configValue != nullptr) { - if (!TypeFormatter::FromString(configValue->GetValue().c_str(), result)) { + if (!configValue->IsValid()) { + MOT_LOG_ERROR("Configuration value of %s is invalid (OOM in string allocation?)", fullPathName); + } else if (!TypeFormatter::FromString(configValue->GetValue().c_str(), result)) { + mot_string stringValue; + MOT_LOG_WARN("Failed to convert string configuration item %s to user type. Using default %s.", + fullPathName, + TypeFormatter::ToString(defaultValue, stringValue)); result = defaultValue; } } @@ -168,7 +176,7 @@ public: inline const T GetIntegerConfigValue(const char* fullPathName, const T& defaultValue) const { const ConfigValue* configValue = GetQualifiedConfigItem(fullPathName); - if (configValue) { + if (configValue != nullptr) { switch (configValue->GetConfigValueType()) { case ConfigValueType::CONFIG_VALUE_INT64: return SafeCast((TypedConfigValue*)configValue, defaultValue); @@ -235,6 +243,38 @@ private: return result; } + /** + * @brief Retrieves a qualified configuration value. Value type is checked according to the template type. + * @param fullPathName The full path name of the configuration array. + * @return The configuration value, or null pointer if none was found by this name, or type does not match. + */ + template + inline const ConfigValue* GetQualifiedConfigValue(const char* fullPathName) const + { + const ConfigValue* result = nullptr; + const ConfigItem* configItem = GetConfigItem(fullPathName); + if (configItem == nullptr) { + MOT_LOG_TRACE("Cannot find configuration item: %s", fullPathName); + } else if (configItem->GetClass() != ConfigItemClass::CONFIG_ITEM_VALUE) { + MOT_LOG_ERROR("Unexpected item class: %s (%s instead of %s)", + fullPathName, + ConfigItemClassToString(configItem->GetClass()), + ConfigItemClassToString(ConfigItemClass::CONFIG_ITEM_VALUE)); + } else { + ConfigValue* configValue = (ConfigValue*)configItem; + if (configValue->GetConfigValueType() != ConfigValueTypeMapper::CONFIG_VALUE_TYPE) { + MOT_LOG_ERROR("Unexpected item value type: %s (%s instead of %s %d)", + fullPathName, + ConfigValueTypeToString(configValue->GetConfigValueType()), + ConfigValueTypeToString(ConfigValueTypeMapper::CONFIG_VALUE_TYPE), + (int)ConfigValueTypeMapper::CONFIG_VALUE_TYPE); + } else { + result = configValue; + } + } + return result; + } + template static inline T SafeCast(TypedConfigValue* configValue, T defaultValue) { diff --git a/src/gausskernel/storage/mot/core/src/infra/config/props_config_file_loader.cpp b/src/gausskernel/storage/mot/core/src/infra/config/props_config_file_loader.cpp index 5e9d1362ad6900dbb1be47bb40057950467c627f..6a35117997611f33c3feead127ead905cfb1b966 100644 --- a/src/gausskernel/storage/mot/core/src/infra/config/props_config_file_loader.cpp +++ b/src/gausskernel/storage/mot/core/src/infra/config/props_config_file_loader.cpp @@ -148,7 +148,7 @@ ConfigTree* PropsConfigFileLoader::LoadConfigFile(const char* configFilePath) configFilePath, reader.GetLineNumber(), line.c_str()); - } else if (!currentSection->AddConfigItem(configItem)) { + } else if (!currentSection->AddConfigItem(configItem, true)) { REPORT_PARSE_ERROR(MOT_ERROR_OOM, "Failed to add configuration item to parent section"); } } else { diff --git a/src/gausskernel/storage/mot/core/src/memory/garbage_collector/mm_gc_manager.cpp b/src/gausskernel/storage/mot/core/src/memory/garbage_collector/mm_gc_manager.cpp index 1aa8fbc471c8885f1eb6c6a2540568392da4c81f..29e4c0168f62dcd97fbdc8828e344e97cee6d976 100644 --- a/src/gausskernel/storage/mot/core/src/memory/garbage_collector/mm_gc_manager.cpp +++ b/src/gausskernel/storage/mot/core/src/memory/garbage_collector/mm_gc_manager.cpp @@ -112,8 +112,8 @@ GcManager* GcManager::Make(int purpose, int threadId, int rcuMaxFreeCount) } else { MOTConfiguration& cfg = GetGlobalConfiguration(); gc->m_isGcEnabled = cfg.m_gcEnable; - gc->m_limboSizeLimit = cfg.m_gcReclaimThresholdBytes; - gc->m_limboSizeLimitHigh = cfg.m_gcHighReclaimThresholdBytes; + gc->m_limboSizeLimit = (uint32_t)cfg.m_gcReclaimThresholdBytes; + gc->m_limboSizeLimitHigh = (uint32_t)cfg.m_gcHighReclaimThresholdBytes; gc->m_rcuFreeCount = cfg.m_gcReclaimBatchSize; if (threadId == 0) { MOT_LOG_INFO( diff --git a/src/gausskernel/storage/mot/core/src/memory/mm_cfg.cpp b/src/gausskernel/storage/mot/core/src/memory/mm_cfg.cpp index dfc96599fddb6b5c2f3ac4069f451f78fe9ca677..159c73a8d75ef2f1219784c6c1164bda40e5af31 100644 --- a/src/gausskernel/storage/mot/core/src/memory/mm_cfg.cpp +++ b/src/gausskernel/storage/mot/core/src/memory/mm_cfg.cpp @@ -41,8 +41,8 @@ DECLARE_LOGGER(MemCfg, Memory) MemCfg g_memGlobalCfg; static int memCfgInitOnce = 0; // initialize only once -static uint32_t totalMemoryMb = 0; -static uint32_t availableMemoryMb = 0; +static uint64_t totalMemoryMb = 0; +static uint64_t availableMemoryMb = 0; static int InitTotalAvailMemory(); static int ValidateCfg(); @@ -78,13 +78,13 @@ extern int MemCfgInit() g_memGlobalCfg.m_minSessionMemoryKb = motCfg.m_sessionMemoryMinLimitKB; g_memGlobalCfg.m_reserveMemoryMode = motCfg.m_reserveMemoryMode; g_memGlobalCfg.m_storeMemoryPolicy = motCfg.m_storeMemoryPolicy; - MOT_LOG_INFO("Global Memory Limit is configured to: %u MB --> %u MB", + MOT_LOG_INFO("Global Memory Limit is configured to: %" PRIu64 " MB --> %" PRIu64 " MB", g_memGlobalCfg.m_minGlobalMemoryMb, g_memGlobalCfg.m_maxGlobalMemoryMb); - MOT_LOG_INFO("Local Memory Limit is configured to: %u MB --> %u MB", + MOT_LOG_INFO("Local Memory Limit is configured to: %" PRIu64 " MB --> %" PRIu64 " MB", g_memGlobalCfg.m_minLocalMemoryMb, g_memGlobalCfg.m_maxLocalMemoryMb); - MOT_LOG_INFO("Session Memory Limit is configured to: %u KB --> %u KB (maximum %u sessions)", + MOT_LOG_INFO("Session Memory Limit is configured to: %" PRIu64 " KB --> %" PRIu64 " KB (maximum %u sessions)", g_memGlobalCfg.m_minSessionMemoryKb, g_memGlobalCfg.m_maxSessionMemoryKb, g_memGlobalCfg.m_maxConnectionCount); @@ -153,12 +153,18 @@ extern void MemCfgToString(int indent, const char* name, StringBuffer* stringBuf indent, "", g_memGlobalCfg.m_lazyLoadChunkDirectory ? "Yes" : "No"); - StringBufferAppend(stringBuffer, "%*sMax Global Memory MB: %u\n", indent, "", g_memGlobalCfg.m_maxGlobalMemoryMb); - StringBufferAppend(stringBuffer, "%*sMin Global Memory MB: %u\n", indent, "", g_memGlobalCfg.m_minGlobalMemoryMb); - StringBufferAppend(stringBuffer, "%*sMax Local Memory MB: %u\n", indent, "", g_memGlobalCfg.m_maxLocalMemoryMb); - StringBufferAppend(stringBuffer, "%*sMin Local Memory MB: %u\n", indent, "", g_memGlobalCfg.m_minLocalMemoryMb); - StringBufferAppend(stringBuffer, "%*sMax Session Memory KB: %u\n", indent, "", g_memGlobalCfg.m_maxSessionMemoryKb); - StringBufferAppend(stringBuffer, "%*sMin Session Memory KB: %u\n", indent, "", g_memGlobalCfg.m_minSessionMemoryKb); + StringBufferAppend( + stringBuffer, "%*sMax Global Memory MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_maxGlobalMemoryMb); + StringBufferAppend( + stringBuffer, "%*sMin Global Memory MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_minGlobalMemoryMb); + StringBufferAppend( + stringBuffer, "%*sMax Local Memory MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_maxLocalMemoryMb); + StringBufferAppend( + stringBuffer, "%*sMin Local Memory MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_minLocalMemoryMb); + StringBufferAppend( + stringBuffer, "%*sMax Session Memory KB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_maxSessionMemoryKb); + StringBufferAppend( + stringBuffer, "%*sMin Session Memory KB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_minSessionMemoryKb); StringBufferAppend(stringBuffer, "%*sReserve Memory Mode: %s\n", indent, @@ -182,25 +188,25 @@ extern void MemCfgToString(int indent, const char* name, StringBuffer* stringBuf g_memGlobalCfg.m_chunkPreallocWorkerCount); StringBufferAppend(stringBuffer, "%*sHigh Red Mark: %u%%\n", indent, "", g_memGlobalCfg.m_highRedMarkPercent); StringBufferAppend(stringBuffer, - "%*sSession Large Buffer Store Size MB: %u\n", + "%*sSession Large Buffer Store Size MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_sessionLargeBufferStoreSizeMb); StringBufferAppend(stringBuffer, - "%*sSession Large Buffer Store Max Object Size MB: %u\n", + "%*sSession Large Buffer Store Max Object Size MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_sessionLargeBufferStoreMaxObjectSizeMb); StringBufferAppend(stringBuffer, - "%*sSession Max Huge Object Size MB: %u\n", + "%*sSession Max Huge Object Size MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_sessionMaxHugeObjectSizeMb); } -extern uint32_t GetTotalSystemMemoryMb() +extern uint64_t GetTotalSystemMemoryMb() { - uint32_t result = 0; + uint64_t result = 0; int res = InitTotalAvailMemory(); if (res != 0) { MOT_REPORT_PANIC(res, "MM Layer Configuration Load", "Failed to load MM Layer configuration"); @@ -210,9 +216,9 @@ extern uint32_t GetTotalSystemMemoryMb() return result; } -extern uint32_t GetAvailableSystemMemoryMb() +extern uint64_t GetAvailableSystemMemoryMb() { - uint32_t result = (uint32_t)-1; + uint64_t result = (uint64_t)-1; int res = InitTotalAvailMemory(); if (res != 0) { MOT_REPORT_PANIC(res, "MM Layer Configuration Load", "Failed to load MM Layer configuration"); @@ -230,8 +236,8 @@ static int InitTotalAvailMemory() if (sysinfo(&si) == 0) { totalMemoryMb = si.totalram * si.mem_unit / MEGA_BYTE; availableMemoryMb = si.freeram * si.mem_unit / MEGA_BYTE; - MOT_LOG_TRACE("Total system memory: %u MB", totalMemoryMb); - MOT_LOG_TRACE("Available system memory: %u MB", availableMemoryMb); + MOT_LOG_TRACE("Total system memory: %" PRIu64 " MB", totalMemoryMb); + MOT_LOG_TRACE("Available system memory: %" PRIu64 " MB", availableMemoryMb); } else { MOT_REPORT_SYSTEM_PANIC( sysinfo, "MM Layer Configuration Load", "Failed to retrieve total/available system memory"); @@ -280,7 +286,8 @@ static int ValidateCfg() if (g_memGlobalCfg.m_minGlobalMemoryMb > g_memGlobalCfg.m_maxGlobalMemoryMb) { MOT_REPORT_PANIC(MOT_ERROR_INVALID_CFG, "MM Layer Configuration Validation", - "Invalid memory configuration: Global memory pre-allocation %u MB exceeds the maximum limit %u MB", + "Invalid memory configuration: Global memory pre-allocation %" PRIu64 + " MB exceeds the maximum limit %" PRIu64 " MB", g_memGlobalCfg.m_minGlobalMemoryMb, g_memGlobalCfg.m_maxGlobalMemoryMb); return MOT_ERROR_INVALID_CFG; @@ -289,7 +296,8 @@ static int ValidateCfg() if (g_memGlobalCfg.m_minLocalMemoryMb > g_memGlobalCfg.m_maxLocalMemoryMb) { MOT_REPORT_PANIC(MOT_ERROR_INVALID_CFG, "MM Layer Configuration Validation", - "Invalid memory configuration: Local memory pre-allocation %u MB exceeds the maximum limit %u MB", + "Invalid memory configuration: Local memory pre-allocation %" PRIu64 + " MB exceeds the maximum limit %" PRIu64 " MB", g_memGlobalCfg.m_minLocalMemoryMb, g_memGlobalCfg.m_maxLocalMemoryMb); return MOT_ERROR_INVALID_CFG; @@ -299,7 +307,8 @@ static int ValidateCfg() if (g_memGlobalCfg.m_maxGlobalMemoryMb > totalMemoryMb) { MOT_REPORT_PANIC(MOT_ERROR_INVALID_CFG, "MM Layer Configuration Validation", - "Invalid memory configuration: Global maximum memory limit %u MB exceeds machine capability %u MB", + "Invalid memory configuration: Global maximum memory limit %" PRIu64 + " MB exceeds machine capability %" PRIu64 " MB", g_memGlobalCfg.m_maxGlobalMemoryMb, totalMemoryMb); return MOT_ERROR_INVALID_CFG; diff --git a/src/gausskernel/storage/mot/core/src/memory/mm_cfg.h b/src/gausskernel/storage/mot/core/src/memory/mm_cfg.h index c074b51635e1067831b3ad958ba52bd927ad9123..8c94da042bf7dc3ff34d9d7850fe6ebe0e6cfb7e 100644 --- a/src/gausskernel/storage/mot/core/src/memory/mm_cfg.h +++ b/src/gausskernel/storage/mot/core/src/memory/mm_cfg.h @@ -42,12 +42,12 @@ struct MemCfg { uint32_t m_lazyLoadChunkDirectory; // memory limits - uint32_t m_maxGlobalMemoryMb; - uint32_t m_minGlobalMemoryMb; - uint32_t m_maxLocalMemoryMb; - uint32_t m_minLocalMemoryMb; - uint32_t m_maxSessionMemoryKb; - uint32_t m_minSessionMemoryKb; + uint64_t m_maxGlobalMemoryMb; + uint64_t m_minGlobalMemoryMb; + uint64_t m_maxLocalMemoryMb; + uint64_t m_minLocalMemoryMb; + uint64_t m_maxSessionMemoryKb; + uint64_t m_minSessionMemoryKb; MemReserveMode m_reserveMemoryMode; MemStorePolicy m_storeMemoryPolicy; @@ -57,9 +57,9 @@ struct MemCfg { uint32_t m_highRedMarkPercent; // session large buffer store - uint32_t m_sessionLargeBufferStoreSizeMb; - uint32_t m_sessionLargeBufferStoreMaxObjectSizeMb; - uint32_t m_sessionMaxHugeObjectSizeMb; + uint64_t m_sessionLargeBufferStoreSizeMb; + uint64_t m_sessionLargeBufferStoreMaxObjectSizeMb; + uint64_t m_sessionMaxHugeObjectSizeMb; }; /** @@ -95,13 +95,13 @@ extern void MemCfgToString(int indent, const char* name, StringBuffer* stringBuf * @brief Retrieves the total system memory in megabytes. * @return The total system memory in megabytes, or zero if failed. */ -extern uint32_t GetTotalSystemMemoryMb(); +extern uint64_t GetTotalSystemMemoryMb(); /** * @brief Retrieves the available system memory in mega-bytes. * @return The available system memory in megabytes, or zero if failed. */ -extern uint32_t GetAvailableSystemMemoryMb(); +extern uint64_t GetAvailableSystemMemoryMb(); } // namespace MOT /** diff --git a/src/gausskernel/storage/mot/core/src/memory/mm_numa.cpp b/src/gausskernel/storage/mot/core/src/memory/mm_numa.cpp index 4c057b014cf26a9532f911dfecbb587b02262868..76d84fe7901856a0b685172399d6977297bc962e 100644 --- a/src/gausskernel/storage/mot/core/src/memory/mm_numa.cpp +++ b/src/gausskernel/storage/mot/core/src/memory/mm_numa.cpp @@ -92,9 +92,10 @@ extern void MemNumaInit() erc = memset_s(peakLocalMemoryBytes, sizeof(peakLocalMemoryBytes), 0, sizeof(peakLocalMemoryBytes)); securec_check(erc, "\0", "\0"); - maxGlobalMemoryBytes = ((uint64_t)g_memGlobalCfg.m_maxGlobalMemoryMb) * MEGABYTE; - maxLocalMemoryBytes = ((uint64_t)g_memGlobalCfg.m_maxLocalMemoryMb) * MEGABYTE; - MOT_LOG_TRACE("Initialized global limit to %u MB (%" PRIu64 " bytes), and local limit to %u MB (%" PRIu64 " bytes)", + maxGlobalMemoryBytes = g_memGlobalCfg.m_maxGlobalMemoryMb * MEGABYTE; + maxLocalMemoryBytes = g_memGlobalCfg.m_maxLocalMemoryMb * MEGABYTE; + MOT_LOG_TRACE("Initialized global limit to %" PRIu64 " MB (%" PRIu64 " bytes), and local limit to %" PRIu64 + " MB (%" PRIu64 " bytes)", g_memGlobalCfg.m_maxGlobalMemoryMb, maxGlobalMemoryBytes, g_memGlobalCfg.m_maxLocalMemoryMb, diff --git a/src/gausskernel/storage/mot/core/src/memory/mm_raw_chunk_store.cpp b/src/gausskernel/storage/mot/core/src/memory/mm_raw_chunk_store.cpp index b28a64c989bd1f53ae721c42ac8c8cff39e2d2e7..ba9016a97bd84881bc2845e91979d9da0142ab07 100644 --- a/src/gausskernel/storage/mot/core/src/memory/mm_raw_chunk_store.cpp +++ b/src/gausskernel/storage/mot/core/src/memory/mm_raw_chunk_store.cpp @@ -318,9 +318,9 @@ static int MemRawChunkStoreInitGlobal(MemReserveMode reserveMode, MemStorePolicy { int result = 0; MOT_LOG_TRACE("Initializing global chunk pools"); - uint32_t chunkReserveCount = g_memGlobalCfg.m_minGlobalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount; - uint32_t highBlackMark = g_memGlobalCfg.m_maxGlobalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount; - uint32_t highRedMark = highBlackMark * g_memGlobalCfg.m_highRedMarkPercent / 100; + uint64_t chunkReserveCount = g_memGlobalCfg.m_minGlobalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount; + uint64_t highBlackMark = g_memGlobalCfg.m_maxGlobalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount; + uint64_t highRedMark = highBlackMark * g_memGlobalCfg.m_highRedMarkPercent / 100; globalChunkPools = (MemRawChunkPool**)calloc(g_memGlobalCfg.m_nodeCount, sizeof(MemRawChunkPool*)); if (globalChunkPools == NULL) { @@ -334,8 +334,8 @@ static int MemRawChunkStoreInitGlobal(MemReserveMode reserveMode, MemStorePolicy reserveMode, storePolicy, chunkReserveCount, - highBlackMark, - highRedMark, + (uint32_t)highBlackMark, + (uint32_t)highRedMark, MEM_ALLOC_GLOBAL); } @@ -365,9 +365,9 @@ static int MemRawChunkStoreInitLocal(MemReserveMode reserveMode, MemStorePolicy { int result = 0; MOT_LOG_TRACE("Initializing local chunk pools"); - uint32_t chunkReserveCount = g_memGlobalCfg.m_minLocalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount; - uint32_t highBlackMark = g_memGlobalCfg.m_maxLocalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount; - uint32_t highRedMark = highBlackMark; // no emergency state for local pools + uint64_t chunkReserveCount = g_memGlobalCfg.m_minLocalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount; + uint64_t highBlackMark = g_memGlobalCfg.m_maxLocalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount; + uint64_t highRedMark = highBlackMark; // no emergency state for local pools localChunkPools = (MemRawChunkPool**)calloc(g_memGlobalCfg.m_nodeCount, sizeof(MemRawChunkPool*)); if (localChunkPools == NULL) { @@ -377,8 +377,13 @@ static int MemRawChunkStoreInitLocal(MemReserveMode reserveMode, MemStorePolicy (unsigned)(g_memGlobalCfg.m_nodeCount * sizeof(MemRawChunkPool*))); result = MOT_ERROR_OOM; } else { - result = InitChunkPools( - localChunkPools, reserveMode, storePolicy, chunkReserveCount, highBlackMark, highRedMark, MEM_ALLOC_LOCAL); + result = InitChunkPools(localChunkPools, + reserveMode, + storePolicy, + chunkReserveCount, + (uint32_t)highBlackMark, + (uint32_t)highRedMark, + MEM_ALLOC_LOCAL); } if (result != 0) { diff --git a/src/gausskernel/storage/mot/core/src/memory/mm_session_api.cpp b/src/gausskernel/storage/mot/core/src/memory/mm_session_api.cpp index 72b454affbf7d906621c52ca284b57f6ca0da826..95bc059c21cc63a223ae320feee8e62e744031de 100644 --- a/src/gausskernel/storage/mot/core/src/memory/mm_session_api.cpp +++ b/src/gausskernel/storage/mot/core/src/memory/mm_session_api.cpp @@ -61,7 +61,7 @@ extern int MemSessionApiInit() { int result = 0; MOT_LOG_TRACE("Initializing session API"); - memSessionHugeBufferMaxObjectSizeBytes = ((uint64_t)g_memGlobalCfg.m_sessionMaxHugeObjectSizeMb) * MEGA_BYTE; + memSessionHugeBufferMaxObjectSizeBytes = g_memGlobalCfg.m_sessionMaxHugeObjectSizeMb * MEGA_BYTE; int rc = pthread_mutex_init(&g_sessionLock, nullptr); if (rc != 0) { diff --git a/src/gausskernel/storage/mot/core/src/mot.conf b/src/gausskernel/storage/mot/core/src/mot.conf index 6c29180d83db0977ad0f5fdc8fe7ac871576a241..ca9eac83181af0b99dbb94576429f1480f190f6e 100644 --- a/src/gausskernel/storage/mot/core/src/mot.conf +++ b/src/gausskernel/storage/mot/core/src/mot.conf @@ -81,6 +81,7 @@ # Specifies the segment size used during checkpoint. # Checkpoint is performed in segments. When a segments is full, it is serialized into disk and a # new segment is opened for the subsequent checkpoint data. +# Note: Percentage values cannot be set for this configuration item. # #checkpoint_segsize = 16 MB @@ -275,7 +276,7 @@ # under which memory is not released back to the kernel, but rather kept in the MOT engine for # later reuse. # -#min_mot_global_memory = 0 MB +#min_mot_global_memory = 0 # Configures the maximum memory limit for the local memory of the MOT engine. # Specifying percentage value relates to the total defined by max_process_memory configured in @@ -297,7 +298,7 @@ # under which memory is not released back to the kernel, but rather kept in the MOT engine for # later reuse. # -#min_mot_local_memory = 0 MB +#min_mot_local_memory = 0 # Configures the maximum memory limit for a single session in MOT engine. # Usually sessions in MOT engine are free to allocate local memory as much as needed, as long as @@ -311,14 +312,14 @@ # max_mot_local_memory. # Note: Percentage values cannot be set for this configuration item. # -#max_mot_session_memory = 0 MB +#max_mot_session_memory = 0 # Configures the minimum memory reservation for a single session in MOT engine. # This value is used both for pre-allocation of memory during session creation, as well as for # ensuring a minimum amount of memory is available for the session to perform its normal operation. # Note: Percentage values cannot be set for this configuration item. # -#min_mot_session_memory = 0 MB +#min_mot_session_memory = 0 # Configures the large buffer store for sessions. # When a user session executes a query that requires a lot of memory (e.g. when using many rows), @@ -329,7 +330,7 @@ # are served directly from kernel. # Note: Percentage values cannot be set for this configuration item. # -#session_large_buffer_store_size = 0 MB +#session_large_buffer_store_size = 0 # Configures the maximum object size in the large allocation buffer store for sessions. # Internally, the large buffer store is divided into objects of various sizes. This value is used @@ -339,7 +340,7 @@ # rectified to the possible maximum. # Note: Percentage values cannot be set for this configuration item. # -#session_large_buffer_store_max_object_size = 0 MB +#session_large_buffer_store_max_object_size = 0 # Configures the maximum size of a single huge memory allocation made by a session. # Huge allocations are served directly from kernel, and therefore are not guaranteed to succeed. diff --git a/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp b/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp index df81fd1f84edf47f3af01a047ebcc1d4724c860d..dd6493d1e036ef8c1955ac1e61d8ccd457c67d0a 100644 --- a/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp +++ b/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp @@ -63,9 +63,9 @@ constexpr bool MOTConfiguration::DEFAULT_ENABLE_CHECKPOINT; constexpr bool MOTConfiguration::DEFAULT_ENABLE_INCREMENTAL_CHECKPOINT; constexpr const char* MOTConfiguration::DEFAULT_CHECKPOINT_DIR; constexpr const char* MOTConfiguration::DEFAULT_CHECKPOINT_SEGSIZE; -constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_SEGSIZE_BYTES; -constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_SEGSIZE_BYTES; -constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_SEGSIZE_BYTES; +constexpr uint64_t MOTConfiguration::DEFAULT_CHECKPOINT_SEGSIZE_BYTES; +constexpr uint64_t MOTConfiguration::MIN_CHECKPOINT_SEGSIZE_BYTES; +constexpr uint64_t MOTConfiguration::MAX_CHECKPOINT_SEGSIZE_BYTES; constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_WORKERS; constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_WORKERS; constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_WORKERS; @@ -82,11 +82,11 @@ constexpr uint16_t MOTConfiguration::DEFAULT_MAX_DATA_NODES; // statistics configuration members constexpr bool MOTConfiguration::DEFAULT_ENABLE_STATS; constexpr const char* MOTConfiguration::DEFAULT_STATS_PRINT_PERIOD; -constexpr uint32_t MOTConfiguration::DEFAULT_STATS_PRINT_PERIOD_SECONDS; -constexpr uint32_t MOTConfiguration::MIN_STATS_PRINT_PERIOD_SECONDS; -constexpr uint32_t MOTConfiguration::MAX_STATS_PRINT_PERIOD_SECONDS; +constexpr uint64_t MOTConfiguration::DEFAULT_STATS_PRINT_PERIOD_SECONDS; +constexpr uint64_t MOTConfiguration::MIN_STATS_PRINT_PERIOD_SECONDS; +constexpr uint64_t MOTConfiguration::MAX_STATS_PRINT_PERIOD_SECONDS; constexpr const char* MOTConfiguration::DEFAULT_FULL_STATS_PRINT_PERIOD; -constexpr uint32_t MOTConfiguration::DEFAULT_FULL_STATS_PRINT_PERIOD_SECONDS; +constexpr uint64_t MOTConfiguration::DEFAULT_FULL_STATS_PRINT_PERIOD_SECONDS; constexpr bool MOTConfiguration::DEFAULT_ENABLE_DB_SESSION_STAT_PRINT; constexpr bool MOTConfiguration::DEFAULT_ENABLE_NETWORK_STAT_PRINT; constexpr bool MOTConfiguration::DEFAULT_ENABLE_LOG_STAT_PRINT; @@ -111,29 +111,29 @@ constexpr uint32_t MOTConfiguration::MAX_MAX_CONNECTIONS; constexpr AffinityMode MOTConfiguration::DEFAULT_AFFINITY_MODE; constexpr bool MOTConfiguration::DEFAULT_LAZY_LOAD_CHUNK_DIRECTORY; constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_GLOBAL_MEMORY; -constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB; -constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_GLOBAL_MEMORY_MB; -constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_GLOBAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::MIN_MAX_MOT_GLOBAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::MAX_MAX_MOT_GLOBAL_MEMORY_MB; constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_LOCAL_MEMORY; -constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_LOCAL_MEMORY_MB; -constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_LOCAL_MEMORY_MB; -constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_LOCAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::DEFAULT_MAX_MOT_LOCAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::MIN_MAX_MOT_LOCAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::MAX_MAX_MOT_LOCAL_MEMORY_MB; constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_GLOBAL_MEMORY; -constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB; -constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_GLOBAL_MEMORY_MB; -constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_GLOBAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::MIN_MIN_MOT_GLOBAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::MAX_MIN_MOT_GLOBAL_MEMORY_MB; constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_LOCAL_MEMORY; -constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_LOCAL_MEMORY_MB; -constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_LOCAL_MEMORY_MB; -constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_LOCAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::DEFAULT_MIN_MOT_LOCAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::MIN_MIN_MOT_LOCAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::MAX_MIN_MOT_LOCAL_MEMORY_MB; constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_SESSION_MEMORY; -constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_SESSION_MEMORY_KB; -constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_SESSION_MEMORY_KB; -constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_SESSION_MEMORY_KB; +constexpr uint64_t MOTConfiguration::DEFAULT_MAX_MOT_SESSION_MEMORY_KB; +constexpr uint64_t MOTConfiguration::MIN_MAX_MOT_SESSION_MEMORY_KB; +constexpr uint64_t MOTConfiguration::MAX_MAX_MOT_SESSION_MEMORY_KB; constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_SESSION_MEMORY; -constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_SESSION_MEMORY_KB; -constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_SESSION_MEMORY_KB; -constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_SESSION_MEMORY_KB; +constexpr uint64_t MOTConfiguration::DEFAULT_MIN_MOT_SESSION_MEMORY_KB; +constexpr uint64_t MOTConfiguration::MIN_MIN_MOT_SESSION_MEMORY_KB; +constexpr uint64_t MOTConfiguration::MAX_MIN_MOT_SESSION_MEMORY_KB; constexpr MemReserveMode MOTConfiguration::DEFAULT_RESERVE_MEMORY_MODE; constexpr MemStorePolicy MOTConfiguration::DEFAULT_STORE_MEMORY_POLICY; constexpr MemAllocPolicy MOTConfiguration::DEFAULT_CHUNK_ALLOC_POLICY; @@ -144,30 +144,30 @@ constexpr uint32_t MOTConfiguration::DEFAULT_HIGH_RED_MARK_PERCENT; constexpr uint32_t MOTConfiguration::MIN_HIGH_RED_MARK_PERCENT; constexpr uint32_t MOTConfiguration::MAX_HIGH_RED_MARK_PERCENT; constexpr const char* MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE; -constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB; -constexpr uint32_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB; -constexpr uint32_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB; +constexpr uint64_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB; +constexpr uint64_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB; +constexpr uint64_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB; constexpr const char* MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE; -constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB; -constexpr uint32_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB; -constexpr uint32_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB; +constexpr uint64_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB; +constexpr uint64_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB; +constexpr uint64_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB; constexpr const char* MOTConfiguration::DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE; -constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB; -constexpr uint32_t MOTConfiguration::MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB; -constexpr uint32_t MOTConfiguration::MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB; +constexpr uint64_t MOTConfiguration::DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB; +constexpr uint64_t MOTConfiguration::MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB; +constexpr uint64_t MOTConfiguration::MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB; // GC configuration members constexpr bool MOTConfiguration::DEFAULT_GC_ENABLE; constexpr const char* MOTConfiguration::DEFAULT_GC_RECLAIM_THRESHOLD; -constexpr uint32_t MOTConfiguration::DEFAULT_GC_RECLAIM_THRESHOLD_BYTES; -constexpr uint32_t MOTConfiguration::MIN_GC_RECLAIM_THRESHOLD_BYTES; -constexpr uint32_t MOTConfiguration::MAX_GC_RECLAIM_THRESHOLD_BYTES; +constexpr uint64_t MOTConfiguration::DEFAULT_GC_RECLAIM_THRESHOLD_BYTES; +constexpr uint64_t MOTConfiguration::MIN_GC_RECLAIM_THRESHOLD_BYTES; +constexpr uint64_t MOTConfiguration::MAX_GC_RECLAIM_THRESHOLD_BYTES; constexpr uint32_t MOTConfiguration::DEFAULT_GC_RECLAIM_BATCH_SIZE; constexpr uint32_t MOTConfiguration::MIN_GC_RECLAIM_BATCH_SIZE; constexpr uint32_t MOTConfiguration::MAX_GC_RECLAIM_BATCH_SIZE; constexpr const char* MOTConfiguration::DEFAULT_GC_HIGH_RECLAIM_THRESHOLD; -constexpr uint32_t MOTConfiguration::DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES; -constexpr uint32_t MOTConfiguration::MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES; -constexpr uint32_t MOTConfiguration::MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES; +constexpr uint64_t MOTConfiguration::DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES; +constexpr uint64_t MOTConfiguration::MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES; +constexpr uint64_t MOTConfiguration::MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES; // JIT configuration members constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_CODEGEN; constexpr bool MOTConfiguration::DEFAULT_FORCE_MOT_PSEUDO_CODEGEN; @@ -184,7 +184,7 @@ constexpr uint64_t MOTConfiguration::DEFAULT_CFG_MONITOR_PERIOD_SECONDS; constexpr uint64_t MOTConfiguration::MIN_CFG_MONITOR_PERIOD_SECONDS; constexpr uint64_t MOTConfiguration::MAX_CFG_MONITOR_PERIOD_SECONDS; constexpr bool MOTConfiguration::DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION; -constexpr uint32_t MOTConfiguration::DEFAULT_TOTAL_MEMORY_MB; +constexpr uint64_t MOTConfiguration::DEFAULT_TOTAL_MEMORY_MB; static constexpr unsigned int MAX_NUMA_NODES = 16u; #define IS_HYPER_THREAD_CMD "lscpu | grep \"Thread(s) per core:\" | awk '{print $4}'" @@ -552,15 +552,15 @@ bool MOTConfiguration::SetFlag(const std::string& name, const std::string& value } else if (ParseBool(name, "enable_checkpoint", value, &m_enableCheckpoint)) { } else if (ParseBool(name, "enable_incremental_checkpoint", value, &m_enableIncrementalCheckpoint)) { } else if (ParseString(name, "checkpoint_dir", value, &m_checkpointDir)) { - } else if (ParseUint32(name, "checkpoint_segsize", value, &m_checkpointSegThreshold)) { + } else if (ParseUint64(name, "checkpoint_segsize", value, &m_checkpointSegThreshold)) { } else if (ParseUint32(name, "checkpoint_workers", value, &m_checkpointWorkers)) { } else if (ParseUint32(name, "checkpoint_recovery_workers", value, &m_checkpointRecoveryWorkers)) { } else if (ParseBool(name, "abort_buffer_enable", value, &m_abortBufferEnable)) { } else if (ParseBool(name, "pre_abort", value, &m_preAbort)) { } else if (ParseValidation(name, "validation_lock", value, &m_validationLock)) { } else if (ParseBool(name, "enable_stats", value, &m_enableStats)) { - } else if (ParseUint32(name, "stats_period_seconds", value, &m_statPrintPeriodSeconds)) { - } else if (ParseUint32(name, "full_stats_period_seconds", value, &m_statPrintFullPeriodSeconds)) { + } else if (ParseUint64(name, "stats_period_seconds", value, &m_statPrintPeriodSeconds)) { + } else if (ParseUint64(name, "full_stats_period_seconds", value, &m_statPrintFullPeriodSeconds)) { } else if (ParseBool(name, "enable_log_recovery_stats", value, &m_enableLogRecoveryStats)) { } else if (ParseBool(name, "enable_db_session_stats", value, &m_enableDbSessionStatistics)) { } else if (ParseBool(name, "enable_network_stats", value, &m_enableNetworkStatistics)) { @@ -579,23 +579,23 @@ bool MOTConfiguration::SetFlag(const std::string& name, const std::string& value } else if (ParseUint32(name, "max_connections", value, &m_maxConnections)) { } else if (ParseAffinity(name, "affinity_mode", value, &m_sessionAffinityMode)) { } else if (ParseBool(name, "lazy_load_chunk_directory", value, &m_lazyLoadChunkDirectory)) { - } else if (ParseUint32(name, "max_mot_global_memory_mb", value, &m_globalMemoryMaxLimitMB)) { - } else if (ParseUint32(name, "min_mot_global_memory_mb", value, &m_globalMemoryMinLimitMB)) { - } else if (ParseUint32(name, "max_mot_local_memory_mb", value, &m_localMemoryMaxLimitMB)) { - } else if (ParseUint32(name, "min_mot_local_memory_mb", value, &m_localMemoryMinLimitMB)) { - } else if (ParseUint32(name, "max_mot_session_memory_kb", value, &m_sessionMemoryMaxLimitKB)) { - } else if (ParseUint32(name, "min_mot_session_memory_kb", value, &m_sessionMemoryMinLimitKB)) { + } else if (ParseUint64(name, "max_mot_global_memory_mb", value, &m_globalMemoryMaxLimitMB)) { + } else if (ParseUint64(name, "min_mot_global_memory_mb", value, &m_globalMemoryMinLimitMB)) { + } else if (ParseUint64(name, "max_mot_local_memory_mb", value, &m_localMemoryMaxLimitMB)) { + } else if (ParseUint64(name, "min_mot_local_memory_mb", value, &m_localMemoryMinLimitMB)) { + } else if (ParseUint64(name, "max_mot_session_memory_kb", value, &m_sessionMemoryMaxLimitKB)) { + } else if (ParseUint64(name, "min_mot_session_memory_kb", value, &m_sessionMemoryMinLimitKB)) { } else if (ParseMemoryReserveMode(name, "reserve_memory_mode", value, &m_reserveMemoryMode)) { } else if (ParseMemoryStorePolicy(name, "store_memory_policy", value, &m_storeMemoryPolicy)) { } else if (ParseChunkAllocPolicy(name, "chunk_alloc_policy", value, &m_chunkAllocPolicy)) { } else if (ParseUint32(name, "chunk_prealloc_worker_count", value, &m_chunkPreallocWorkerCount)) { } else if (ParseUint32(name, "high_red_mark_percent", value, &m_highRedMarkPercent)) { - } else if (ParseUint32(name, "session_large_buffer_store_size_mb", value, &m_sessionLargeBufferStoreSizeMB)) { - } else if (ParseUint32(name, + } else if (ParseUint64(name, "session_large_buffer_store_size_mb", value, &m_sessionLargeBufferStoreSizeMB)) { + } else if (ParseUint64(name, "session_large_buffer_store_max_object_size_mb", value, &m_sessionLargeBufferStoreMaxObjectSizeMB)) { - } else if (ParseUint32(name, "session_max_huge_object_size_mb", value, &m_sessionMaxHugeObjectSizeMB)) { + } else if (ParseUint64(name, "session_max_huge_object_size_mb", value, &m_sessionMaxHugeObjectSizeMB)) { } else if (ParseBool(name, "enable_mot_codegen", value, &m_enableCodegen)) { } else if (ParseBool(name, "force_mot_pseudo_codegen", value, &m_forcePseudoCodegen)) { } else if (ParseBool(name, "enable_mot_codegen_print", value, &m_enableCodegenPrint)) { @@ -675,26 +675,14 @@ int MOTConfiguration::GetMappedCore(int logicId) const #define UPDATE_INT_CFG(var, cfgPath, defaultValue, lowerBound, upperBound) \ UpdateIntConfigItem(var, cfg->GetIntegerConfigValue(cfgPath, defaultValue), cfgPath, lowerBound, upperBound) -#define UPDATE_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \ - do { \ - uint64_t memoryValueBytes = \ - ParseMemoryValueBytes(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \ - UpdateIntConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath, lowerBound, upperBound); \ - } while (0); - -#define UPDATE_ABS_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \ - do { \ - uint64_t memoryValueBytes = \ - ParseMemoryUnit(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \ - UpdateIntConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath, lowerBound, upperBound); \ - } while (0); - -#define UPDATE_TIME_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \ - do { \ - uint64_t timeValueMicros = \ - ParseTimeValueMicros(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \ - UpdateIntConfigItem(var, (uint64_t)(timeValueMicros / scale), cfgPath, lowerBound, upperBound); \ - } while (0); +#define UPDATE_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \ + UpdateMemConfigItem(var, cfgPath, defaultValue, scale, lowerBound, upperBound, true) + +#define UPDATE_ABS_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \ + UpdateMemConfigItem(var, cfgPath, defaultValue, scale, lowerBound, upperBound, false) + +#define UPDATE_TIME_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \ + UpdateTimeConfigItem(var, cfgPath, defaultValue, scale, lowerBound, upperBound) void MOTConfiguration::LoadConfig() { @@ -728,7 +716,7 @@ void MOTConfiguration::LoadConfig() // Checkpoint configuration UPDATE_BOOL_CFG(m_enableCheckpoint, "enable_checkpoint", DEFAULT_ENABLE_CHECKPOINT); UPDATE_STRING_CFG(m_checkpointDir, "checkpoint_dir", DEFAULT_CHECKPOINT_DIR); - UPDATE_MEM_CFG(m_checkpointSegThreshold, + UPDATE_ABS_MEM_CFG(m_checkpointSegThreshold, "checkpoint_segsize", DEFAULT_CHECKPOINT_SEGSIZE, SCALE_BYTES, @@ -810,15 +798,24 @@ void MOTConfiguration::LoadConfig() // validate that min <= max if (m_globalMemoryMinLimitMB > m_globalMemoryMaxLimitMB) { if (!m_suppressLog) { - MOT_LOG_WARN("Invalid global memory configuration: minimum (%u MB) is greater than maximum (%u MB), using " - "defaults (%u MB, %u MB)", + MOT_LOG_WARN("Invalid global memory configuration: minimum (%" PRIu64 + " MB) is greater than maximum (%" PRIu64 " MB), using defaults (%" PRIu64 " MB, %" PRIu64 + " MB)", m_globalMemoryMinLimitMB, m_globalMemoryMaxLimitMB, DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB, DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB); } - UpdateIntConfigItem(m_globalMemoryMaxLimitMB, DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB, "max_mot_global_memory"); - UpdateIntConfigItem(m_globalMemoryMinLimitMB, DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB, "min_mot_global_memory"); + UpdateIntConfigItem(m_globalMemoryMaxLimitMB, + DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB, + "max_mot_global_memory", + MIN_MAX_MOT_GLOBAL_MEMORY_MB, + MAX_MAX_MOT_GLOBAL_MEMORY_MB); + UpdateIntConfigItem(m_globalMemoryMinLimitMB, + DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB, + "min_mot_global_memory", + MIN_MIN_MOT_GLOBAL_MEMORY_MB, + MAX_MIN_MOT_GLOBAL_MEMORY_MB); } UPDATE_MEM_CFG(m_localMemoryMaxLimitMB, "max_mot_local_memory", @@ -835,15 +832,24 @@ void MOTConfiguration::LoadConfig() // validate that min <= max if (m_localMemoryMinLimitMB > m_localMemoryMaxLimitMB) { if (!m_suppressLog) { - MOT_LOG_WARN("Invalid local memory configuration: minimum (%u MB) is greater than maximum (%u MB), using " - "defaults (%u MB, %u MB)", + MOT_LOG_WARN("Invalid local memory configuration: minimum (%" PRIu64 + " MB) is greater than maximum (%" PRIu64 " MB), using defaults (%" PRIu64 " MB, %" PRIu64 + " MB)", m_localMemoryMinLimitMB, m_localMemoryMaxLimitMB, DEFAULT_MIN_MOT_LOCAL_MEMORY_MB, DEFAULT_MAX_MOT_LOCAL_MEMORY_MB); } - UpdateIntConfigItem(m_localMemoryMaxLimitMB, DEFAULT_MAX_MOT_LOCAL_MEMORY_MB, "max_mot_local_memory"); - UpdateIntConfigItem(m_localMemoryMinLimitMB, DEFAULT_MIN_MOT_LOCAL_MEMORY_MB, "min_mot_local_memory"); + UpdateIntConfigItem(m_localMemoryMaxLimitMB, + DEFAULT_MAX_MOT_LOCAL_MEMORY_MB, + "max_mot_local_memory", + MIN_MAX_MOT_LOCAL_MEMORY_MB, + MAX_MAX_MOT_LOCAL_MEMORY_MB); + UpdateIntConfigItem(m_localMemoryMinLimitMB, + DEFAULT_MIN_MOT_LOCAL_MEMORY_MB, + "min_mot_local_memory", + MIN_MIN_MOT_LOCAL_MEMORY_MB, + MAX_MIN_MOT_LOCAL_MEMORY_MB); } UPDATE_ABS_MEM_CFG(m_sessionMemoryMaxLimitKB, "max_mot_session_memory", @@ -860,15 +866,24 @@ void MOTConfiguration::LoadConfig() // validate that min <= max if ((m_sessionMemoryMaxLimitKB > 0) && (m_sessionMemoryMinLimitKB > m_sessionMemoryMaxLimitKB)) { if (!m_suppressLog) { - MOT_LOG_WARN("Invalid session memory configuration: minimum (%u KB) is greater than maximum (%u KB), using " - "defaults (%u KB, %u KB)", + MOT_LOG_WARN("Invalid session memory configuration: minimum (%" PRIu64 + " KB) is greater than maximum (%" PRIu64 " KB), using defaults (%" PRIu64 " KB, %" PRIu64 + " KB)", m_sessionMemoryMinLimitKB, m_sessionMemoryMaxLimitKB, DEFAULT_MIN_MOT_SESSION_MEMORY_KB, DEFAULT_MAX_MOT_SESSION_MEMORY_KB); } - UpdateIntConfigItem(m_sessionMemoryMaxLimitKB, DEFAULT_MAX_MOT_SESSION_MEMORY_KB, "max_mot_session_memory"); - UpdateIntConfigItem(m_sessionMemoryMinLimitKB, DEFAULT_MIN_MOT_SESSION_MEMORY_KB, "min_mot_session_memory"); + UpdateIntConfigItem(m_sessionMemoryMaxLimitKB, + DEFAULT_MAX_MOT_SESSION_MEMORY_KB, + "max_mot_session_memory", + MIN_MAX_MOT_SESSION_MEMORY_KB, + MAX_MAX_MOT_SESSION_MEMORY_KB); + UpdateIntConfigItem(m_sessionMemoryMinLimitKB, + DEFAULT_MIN_MOT_SESSION_MEMORY_KB, + "min_mot_session_memory", + MIN_MIN_MOT_SESSION_MEMORY_KB, + MAX_MIN_MOT_SESSION_MEMORY_KB); } UPDATE_USER_CFG(m_reserveMemoryMode, "reserve_memory_mode", DEFAULT_RESERVE_MEMORY_MODE); UPDATE_USER_CFG(m_storeMemoryPolicy, "store_memory_policy", DEFAULT_STORE_MEMORY_POLICY); @@ -951,6 +966,101 @@ void MOTConfiguration::LoadConfig() MOT_LOG_TRACE("Main configuration loaded"); } +void MOTConfiguration::UpdateMemConfigItem(uint64_t& oldValue, const char* name, const char* defaultStrValue, + uint64_t scale, uint64_t lowerBound, uint64_t upperBound, bool allowPercentage) +{ + // we prepare first a default value from the string default value + uint64_t defaultValueBytes = allowPercentage ? ParseMemoryValueBytes(defaultStrValue, (uint64_t)-1, name) + : ParseMemoryUnit(defaultStrValue, (uint64_t)-1, name); + MOT_LOG_TRACE( + "Converted memory default string value %s to byte value: %" PRIu64, defaultStrValue, defaultValueBytes); + + // now we carefully examine the configuration item type + const LayeredConfigTree* cfg = ConfigManager::GetInstance().GetLayeredConfigTree(); + const ConfigItem* cfgItem = cfg->GetConfigItem(name); + if (cfgItem == nullptr) { + // just keep default + MOT_LOG_TRACE("Configuration item %s not found, keeping default %" PRIu64 ".", name, defaultValueBytes); + } else if (cfgItem->GetClass() != ConfigItemClass::CONFIG_ITEM_VALUE) { + MOT_LOG_ERROR("Invalid configuration for %s: expected value item, got %s item. Keeping default value %" PRIu64 + ".", + name, + ConfigItemClassToString(cfgItem->GetClass()), + defaultValueBytes); + UpdateIntConfigItem(oldValue, defaultValueBytes / scale, name, lowerBound, upperBound); + } else { + // now we carefully examine the value type + ConfigValue* cfgValue = (ConfigValue*)cfgItem; + if (cfgValue->IsIntegral()) { + // only a number was specified, so it is interpreted as bytes + uint64_t memoryValueBytes = cfg->GetIntegerConfigValue(name, defaultValueBytes); + UpdateIntConfigItem(oldValue, memoryValueBytes / scale, name, lowerBound, upperBound); + MOT_LOG_TRACE("Loading integral memory value %s as bytes: %" PRIu64, name, memoryValueBytes); + } else if (cfgValue->GetConfigValueType() == ConfigValueType::CONFIG_VALUE_STRING) { + // value was parsed as string, meaning we have units or percentage specifier + const char* strValue = cfg->GetStringConfigValue(name, defaultStrValue); + if ((strValue != nullptr) && (strValue[0] != 0)) { + uint64_t memoryValueBytes = allowPercentage ? ParseMemoryValueBytes(strValue, (uint64_t)-1, name) + : ParseMemoryUnit(strValue, (uint64_t)-1, name); + UpdateIntConfigItem(oldValue, memoryValueBytes / scale, name, lowerBound, upperBound); + } + } else { + // unexpected value type + MOT_LOG_WARN("Unexpected configuration item %s value type: %s. Keeping default value: %" PRIu64 ".", + name, + ConfigValueTypeToString(cfgValue->GetConfigValueType()), + defaultValueBytes); + UpdateIntConfigItem(oldValue, defaultValueBytes / scale, name, lowerBound, upperBound); + } + } +} + +void MOTConfiguration::UpdateTimeConfigItem(uint64_t& oldValue, const char* name, const char* defaultStrValue, + uint64_t scale, uint64_t lowerBound, uint64_t upperBound) +{ + // we prepare first a default value from the string default value + uint64_t defaultValueUSecs = ParseTimeValueMicros(defaultStrValue, (uint64_t)-1, name); + MOT_LOG_TRACE("Converted time default string value %s to usec value: %" PRIu64, defaultStrValue, defaultValueUSecs); + + // now we carefully examine the configuration item type + const LayeredConfigTree* cfg = ConfigManager::GetInstance().GetLayeredConfigTree(); + const ConfigItem* cfgItem = cfg->GetConfigItem(name); + if (cfgItem == nullptr) { + // just keep default + MOT_LOG_TRACE("Configuration item %s not found, keeping default %" PRIu64 ".", name, defaultValueUSecs); + } else if (cfgItem->GetClass() != ConfigItemClass::CONFIG_ITEM_VALUE) { + MOT_LOG_ERROR("Invalid configuration for %s: expected value item, got %s item. Keeping default value %" PRIu64 + ".", + name, + ConfigItemClassToString(cfgItem->GetClass()), + defaultValueUSecs); + UpdateIntConfigItem(oldValue, defaultValueUSecs / scale, name, lowerBound, upperBound); + } else { + // now we carefully examine the value type + ConfigValue* cfgValue = (ConfigValue*)cfgItem; + if (cfgValue->IsIntegral()) { + // only a number was specified, so it is interpreted as micro-seconds + uint64_t timeValueUSecs = cfg->GetIntegerConfigValue(name, defaultValueUSecs); + UpdateIntConfigItem(oldValue, timeValueUSecs / scale, name, lowerBound, upperBound); + MOT_LOG_TRACE("Loading integral time value %s as micro-seconds: %" PRIu64, name, timeValueUSecs); + } else if (cfgValue->GetConfigValueType() == ConfigValueType::CONFIG_VALUE_STRING) { + // value was parsed as string, meaning we have units + const char* strValue = cfg->GetStringConfigValue(name, defaultStrValue); + if ((strValue != nullptr) && (strValue[0] != 0)) { + uint64_t timeValueUSecs = ParseTimeValueMicros(strValue, (uint64_t)-1, name); + UpdateIntConfigItem(oldValue, timeValueUSecs / scale, name, lowerBound, upperBound); + } + } else { + // unexpected value type + MOT_LOG_WARN("Unexpected configuration item %s value type: %s. Keeping default value: %" PRIu64 ".", + name, + ConfigValueTypeToString(cfgValue->GetConfigValueType()), + defaultValueUSecs); + UpdateIntConfigItem(oldValue, defaultValueUSecs / scale, name, lowerBound, upperBound); + } + } +} + void MOTConfiguration::UpdateComponentLogLevel() { LogLevel globalLogLevel = GetGlobalLogLevel(); @@ -1083,7 +1193,7 @@ uint64_t MOTConfiguration::ParseMemoryPercentTotal(const char* memoryValue, uint // we expect to parse a number between 0-100, and then followed by "%" int percent = ParseMemoryPercent(memoryValue); if (percent >= 0) { - memoryValueBytes = ((uint64_t)m_totalMemoryMb) * MEGA_BYTE * percent / 100; + memoryValueBytes = m_totalMemoryMb * MEGA_BYTE * percent / 100; if (!m_suppressLog) { MOT_LOG_INFO( "Loaded %s: %d%% from total = %" PRIu64 " MB", cfgPath, percent, memoryValueBytes / 1024ul / 1024ul); @@ -1102,7 +1212,8 @@ uint64_t MOTConfiguration::ParseMemoryUnit(const char* memoryValue, uint64_t def if (endptr == memoryValue) { MOT_LOG_WARN("Invalid %s memory value format: %s (expecting value digits)", cfgPath, memoryValue); } else if (*endptr == 0) { - MOT_LOG_WARN("Invalid %s memory value format: %s (expecting unit type after value)", cfgPath, memoryValue); + MOT_LOG_TRACE("Missing %s memory value units: bytes assumed", cfgPath, memoryValue); + memoryValueBytes = value; } else { // get unit type and convert to mega-bytes mot_string suffix(endptr); @@ -1146,26 +1257,32 @@ uint64_t MOTConfiguration::ParseTimeValueMicros(const char* timeValue, uint64_t // get unit type and convert to mega-bytes mot_string suffix(endptr); suffix.trim(); - if ((suffix.compare_no_case("d") == 0) || (suffix.compare_no_case("days") == 0)) { + if ((suffix.compare_no_case("d") == 0) || (suffix.compare_no_case("days") == 0) || + (suffix.compare_no_case("day") == 0)) { MOT_LOG_TRACE("Loaded %s: %u days", cfgPath, value); timeValueMicors = ((uint64_t)value) * 24ull * 60ull * 60ull * 1000ull * 1000ull; - } else if ((suffix.compare_no_case("h") == 0) || (suffix.compare_no_case("hours") == 0)) { + } else if ((suffix.compare_no_case("h") == 0) || (suffix.compare_no_case("hours") == 0) || + (suffix.compare_no_case("hour") == 0)) { MOT_LOG_TRACE("Loaded %s: %u hours", cfgPath, value); timeValueMicors = ((uint64_t)value) * 60ull * 60ull * 1000ull * 1000ull; } else if ((suffix.compare_no_case("m") == 0) || (suffix.compare_no_case("mins") == 0) || - (suffix.compare_no_case("minutes") == 0)) { + (suffix.compare_no_case("minutes") == 0) || (suffix.compare_no_case("min") == 0) || + (suffix.compare_no_case("minute") == 0)) { MOT_LOG_TRACE("Loaded %s: %u minutes", cfgPath, value); timeValueMicors = ((uint64_t)value) * 60ull * 1000ull * 1000ull; } else if ((suffix.compare_no_case("s") == 0) || (suffix.compare_no_case("secs") == 0) || - (suffix.compare_no_case("seconds") == 0)) { + (suffix.compare_no_case("seconds") == 0) || (suffix.compare_no_case("sec") == 0) || + (suffix.compare_no_case("second") == 0)) { MOT_LOG_TRACE("Loaded %s: %u seconds", cfgPath, value); timeValueMicors = ((uint64_t)value) * 1000ull * 1000ull; } else if ((suffix.compare_no_case("ms") == 0) || (suffix.compare_no_case("millis") == 0) || - (suffix.compare_no_case("milliseocnds") == 0)) { + (suffix.compare_no_case("milliseconds") == 0) || (suffix.compare_no_case("milli") == 0) || + (suffix.compare_no_case("millisecond") == 0)) { MOT_LOG_TRACE("Loaded %s: %u milli-seconds", cfgPath, value); timeValueMicors = ((uint64_t)value) * 1000ull; } else if ((suffix.compare_no_case("us") == 0) || (suffix.compare_no_case("micros") == 0) || - (suffix.compare_no_case("microseocnds") == 0)) { + (suffix.compare_no_case("microseconds") == 0) || (suffix.compare_no_case("micro") == 0) || + (suffix.compare_no_case("microsecond") == 0)) { MOT_LOG_TRACE("Loaded %s: %u micro-seconds", cfgPath, value); timeValueMicors = ((uint64_t)value); } else { diff --git a/src/gausskernel/storage/mot/core/src/system/mot_configuration.h b/src/gausskernel/storage/mot/core/src/system/mot_configuration.h index 828483b2a0cf4d610a9228cdb4d390af7344a9f7..980ed3f8a91b0395b28df86c56a1fd4495719496 100644 --- a/src/gausskernel/storage/mot/core/src/system/mot_configuration.h +++ b/src/gausskernel/storage/mot/core/src/system/mot_configuration.h @@ -107,9 +107,9 @@ public: } /** @brief Configures the total memory used as reference for computing memory percentage values. */ - inline void SetTotalMemoryMb(uint32_t totalMemoryMb) + inline void SetTotalMemoryMb(uint64_t totalMemoryMb) { - MOT_LOG_INFO("Configuring total memory for relative memory values to: %u MB", totalMemoryMb); + MOT_LOG_INFO("Configuring total memory for relative memory values to: %" PRIu64 " MB", totalMemoryMb); m_totalMemoryMb = totalMemoryMb; } @@ -167,7 +167,7 @@ public: std::string m_checkpointDir; /** @var checkpoint segments size in bytes. */ - uint32_t m_checkpointSegThreshold; + uint64_t m_checkpointSegThreshold; /** @var number of worker threads to spawn to perform checkpoint. */ uint32_t m_checkpointWorkers; @@ -209,10 +209,10 @@ public: bool m_enableStats; /** Statistics printing period seconds. */ - uint32_t m_statPrintPeriodSeconds; + uint64_t m_statPrintPeriodSeconds; /** Full statistics printing period in seconds. */ - uint32_t m_statPrintFullPeriodSeconds; + uint64_t m_statPrintFullPeriodSeconds; /** @var Enable statistics gathering and printing during redo log recovery. */ bool m_enableLogRecoveryStats; @@ -278,22 +278,22 @@ public: bool m_lazyLoadChunkDirectory; /** @var Maximum global memory limit in megabytes. */ - uint32_t m_globalMemoryMaxLimitMB; + uint64_t m_globalMemoryMaxLimitMB; /** @var Minimum global memory limit in megabytes (implies pre-allocation and minimum reservation). */ - uint32_t m_globalMemoryMinLimitMB; + uint64_t m_globalMemoryMinLimitMB; /** @var Maximum local (per-node) memory limit in megabytes. */ - uint32_t m_localMemoryMaxLimitMB; + uint64_t m_localMemoryMaxLimitMB; /** @var Minimum local (per-node) memory limit in megabytes (implies pre-allocation and minimum reservation). */ - uint32_t m_localMemoryMinLimitMB; + uint64_t m_localMemoryMinLimitMB; /** @var Maximum for single MOT session small memory allocations. */ - uint32_t m_sessionMemoryMaxLimitKB; + uint64_t m_sessionMemoryMaxLimitKB; /** @var Minimum (pre-allocated) for single MOT session small memory allocations. */ - uint32_t m_sessionMemoryMinLimitKB; + uint64_t m_sessionMemoryMinLimitKB; /* @var Specifies whether to reserve physical memory on startup (or just virtual). */ MemReserveMode m_reserveMemoryMode; @@ -311,13 +311,13 @@ public: uint32_t m_highRedMarkPercent; /** @var The size in megabytes of the session large buffer store. */ - uint32_t m_sessionLargeBufferStoreSizeMB; + uint64_t m_sessionLargeBufferStoreSizeMB; /** @var The largest object size in megabytes in the session large buffer store. */ - uint32_t m_sessionLargeBufferStoreMaxObjectSizeMB; + uint64_t m_sessionLargeBufferStoreMaxObjectSizeMB; /** @var The largest single huge object size that can be allocated by any session directly from kernel. */ - uint32_t m_sessionMaxHugeObjectSizeMB; + uint64_t m_sessionMaxHugeObjectSizeMB; /**********************************************************************/ // Garbage Collection configuration @@ -326,13 +326,13 @@ public: bool m_gcEnable; /** @var The threshold in bytes for reclamation to be triggered (per-thread). */ - uint32_t m_gcReclaimThresholdBytes; + uint64_t m_gcReclaimThresholdBytes; /** @var The amount of objects reclaimed in each cleanup round of a limbo group. */ uint32_t m_gcReclaimBatchSize; /** @var The high threshold in bytes for reclamation to be triggered (per-thread). */ - uint32_t m_gcHighReclaimThresholdBytes; + uint64_t m_gcHighReclaimThresholdBytes; /**********************************************************************/ // JIT configuration @@ -411,7 +411,7 @@ private: bool m_isSystemHyperThreaded = false; /** @var The total memory value to be used when loading memory percent values. By default uses system total. */ - uint32_t m_totalMemoryMb; + uint64_t m_totalMemoryMb; /** @var Controls suppressing of log messages during configuration loading. */ bool m_suppressLog; @@ -467,9 +467,9 @@ private: /** @var Default checkpoint segments size */ static constexpr const char* DEFAULT_CHECKPOINT_SEGSIZE = "16 MB"; - static constexpr uint32_t DEFAULT_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE; - static constexpr uint32_t MIN_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE; - static constexpr uint32_t MAX_CHECKPOINT_SEGSIZE_BYTES = 512 * MEGA_BYTE; + static constexpr uint64_t DEFAULT_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE; + static constexpr uint64_t MIN_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE; + static constexpr uint64_t MAX_CHECKPOINT_SEGSIZE_BYTES = 512 * MEGA_BYTE; /** @var Default number of worker threads to spawn */ static constexpr uint32_t DEFAULT_CHECKPOINT_WORKERS = 3; @@ -504,13 +504,13 @@ private: /** @var Default statistics printing period in seconds. */ static constexpr const char* DEFAULT_STATS_PRINT_PERIOD = "1 minutes"; - static constexpr uint32_t DEFAULT_STATS_PRINT_PERIOD_SECONDS = 60; - static constexpr uint32_t MIN_STATS_PRINT_PERIOD_SECONDS = 1; - static constexpr uint32_t MAX_STATS_PRINT_PERIOD_SECONDS = 86400; // 1 day + static constexpr uint64_t DEFAULT_STATS_PRINT_PERIOD_SECONDS = 60; + static constexpr uint64_t MIN_STATS_PRINT_PERIOD_SECONDS = 1; + static constexpr uint64_t MAX_STATS_PRINT_PERIOD_SECONDS = 86400; // 1 day /** @var Default full statistics printing period in seconds. */ static constexpr const char* DEFAULT_FULL_STATS_PRINT_PERIOD = "5 minutes"; - static constexpr uint32_t DEFAULT_FULL_STATS_PRINT_PERIOD_SECONDS = 300; + static constexpr uint64_t DEFAULT_FULL_STATS_PRINT_PERIOD_SECONDS = 300; /** @var Default enable DB session statistics printing. */ static constexpr bool DEFAULT_ENABLE_DB_SESSION_STAT_PRINT = false; @@ -571,40 +571,40 @@ private: /** @var Default maximum limit for MOT global memory. */ static constexpr const char* DEFAULT_MAX_MOT_GLOBAL_MEMORY = "80%"; - static constexpr uint32_t DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB = 8 * KILO_BYTE; // 8 GB - static constexpr uint32_t MIN_MAX_MOT_GLOBAL_MEMORY_MB = 128; // 128 MB - static constexpr uint32_t MAX_MAX_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB + static constexpr uint64_t DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB = 8 * KILO_BYTE; // 8 GB + static constexpr uint64_t MIN_MAX_MOT_GLOBAL_MEMORY_MB = 128; // 128 MB + static constexpr uint64_t MAX_MAX_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB /** @var Default minimum (pre-allocated) limit for MOT global memory. */ - static constexpr const char* DEFAULT_MIN_MOT_GLOBAL_MEMORY = "0 GB"; - static constexpr uint32_t DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB = 0; - static constexpr uint32_t MIN_MIN_MOT_GLOBAL_MEMORY_MB = 0; - static constexpr uint32_t MAX_MIN_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB + static constexpr const char* DEFAULT_MIN_MOT_GLOBAL_MEMORY = "0"; + static constexpr uint64_t DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB = 0; + static constexpr uint64_t MIN_MIN_MOT_GLOBAL_MEMORY_MB = 0; + static constexpr uint64_t MAX_MIN_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB /** @var Default maximum limit for MOT global memory (used to establish new ratio between local and global pools). */ static constexpr const char* DEFAULT_MAX_MOT_LOCAL_MEMORY = "15%"; - static constexpr uint32_t DEFAULT_MAX_MOT_LOCAL_MEMORY_MB = 2 * KILO_BYTE; // 2 GB - static constexpr uint32_t MIN_MAX_MOT_LOCAL_MEMORY_MB = 64; // 64 MB (about 8 normal sessions) - static constexpr uint32_t MAX_MAX_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // 512 GB (MANY very heavy sessions) + static constexpr uint64_t DEFAULT_MAX_MOT_LOCAL_MEMORY_MB = 2 * KILO_BYTE; // 2 GB + static constexpr uint64_t MIN_MAX_MOT_LOCAL_MEMORY_MB = 64; // 64 MB (about 8 normal sessions) + static constexpr uint64_t MAX_MAX_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // 512 GB (MANY very heavy sessions) /** @var Default minimum (pre-allocated) limit for MOT local memory. */ - static constexpr const char* DEFAULT_MIN_MOT_LOCAL_MEMORY = "0 GB"; - static constexpr uint32_t DEFAULT_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation - static constexpr uint32_t MIN_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation - static constexpr uint32_t MAX_MIN_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // max pre-allocate 512 GB + static constexpr const char* DEFAULT_MIN_MOT_LOCAL_MEMORY = "0"; + static constexpr uint64_t DEFAULT_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation + static constexpr uint64_t MIN_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation + static constexpr uint64_t MAX_MIN_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // max pre-allocate 512 GB /** @var Default maximum for single MOT session small memory allocations. */ - static constexpr const char* DEFAULT_MAX_MOT_SESSION_MEMORY = "0 MB"; - static constexpr uint32_t DEFAULT_MAX_MOT_SESSION_MEMORY_KB = 0; // no session-memory limit - static constexpr uint32_t MIN_MAX_MOT_SESSION_MEMORY_KB = 0; - static constexpr uint32_t MAX_MAX_MOT_SESSION_MEMORY_KB = 512 * MEGA_BYTE; // limit single session to 512 GB + static constexpr const char* DEFAULT_MAX_MOT_SESSION_MEMORY = "0"; + static constexpr uint64_t DEFAULT_MAX_MOT_SESSION_MEMORY_KB = 0; // no session-memory limit + static constexpr uint64_t MIN_MAX_MOT_SESSION_MEMORY_KB = 0; + static constexpr uint64_t MAX_MAX_MOT_SESSION_MEMORY_KB = 512 * MEGA_BYTE; // limit single session to 512 GB /** @var Default minimum (pre-allocated) for single MOT session small memory allocations. */ - static constexpr const char* DEFAULT_MIN_MOT_SESSION_MEMORY = "0 MB"; - static constexpr uint32_t DEFAULT_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation - static constexpr uint32_t MIN_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation - static constexpr uint32_t MAX_MIN_MOT_SESSION_MEMORY_KB = 64 * KILO_BYTE; // up to 64 MB pre-allocation + static constexpr const char* DEFAULT_MIN_MOT_SESSION_MEMORY = "0"; + static constexpr uint64_t DEFAULT_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation + static constexpr uint64_t MIN_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation + static constexpr uint64_t MAX_MIN_MOT_SESSION_MEMORY_KB = 64 * KILO_BYTE; // up to 64 MB pre-allocation /** @var Default physical or virtual memory reservation. */ static constexpr MemReserveMode DEFAULT_RESERVE_MEMORY_MODE = MEM_RESERVE_VIRTUAL; @@ -626,22 +626,22 @@ private: static constexpr uint32_t MAX_HIGH_RED_MARK_PERCENT = 95; // Don't allow disabling high red-mark /** @var The default size in megabytes of the session large buffer store. */ - static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE = " 0 MB"; - static constexpr uint32_t DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0; - static constexpr uint32_t MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0; // disabled - static constexpr uint32_t MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 128 * KILO_BYTE; // 128 GB + static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE = "0"; + static constexpr uint64_t DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0; + static constexpr uint64_t MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0; // disabled + static constexpr uint64_t MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 128 * KILO_BYTE; // 128 GB /** @var The default largest object size in megabytes in the session large buffer store. */ - static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE = "0 MB"; - static constexpr uint32_t DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0; // use calculation - static constexpr uint32_t MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0; - static constexpr uint32_t MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 1024; // 1 GB + static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE = "0"; + static constexpr uint64_t DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0; // use calculation + static constexpr uint64_t MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0; + static constexpr uint64_t MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 1024; // 1 GB /** @var The default largest object size in megabytes that can be allocated form kernel for sessions. */ static constexpr const char* DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE = "1 GB"; - static constexpr uint32_t DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 1024; // 1 GB - static constexpr uint32_t MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8; // 8 MB - static constexpr uint32_t MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8 * KILO_BYTE; // 8 GB + static constexpr uint64_t DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 1024; // 1 GB + static constexpr uint64_t MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8; // 8 MB + static constexpr uint64_t MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8 * KILO_BYTE; // 8 GB /** ------------------ Default Garbage-Collection Configuration ------------ */ /** @var Enable/disable garbage collection. */ @@ -649,9 +649,9 @@ private: /** @var The threshold in bytes for reclamation to be triggered (per-thread) */ static constexpr const char* DEFAULT_GC_RECLAIM_THRESHOLD = "512 KB"; - static constexpr uint32_t DEFAULT_GC_RECLAIM_THRESHOLD_BYTES = 512 * KILO_BYTE; // 512 KB - static constexpr uint32_t MIN_GC_RECLAIM_THRESHOLD_BYTES = KILO_BYTE; // 1 KB - static constexpr uint32_t MAX_GC_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB + static constexpr uint64_t DEFAULT_GC_RECLAIM_THRESHOLD_BYTES = 512 * KILO_BYTE; // 512 KB + static constexpr uint64_t MIN_GC_RECLAIM_THRESHOLD_BYTES = KILO_BYTE; // 1 KB + static constexpr uint64_t MAX_GC_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB /** @var The amount of objects reclaimed in each cleanup round of a limbo group. */ static constexpr uint32_t DEFAULT_GC_RECLAIM_BATCH_SIZE = 8 * KILO_BYTE; // 8 KB entries @@ -660,9 +660,9 @@ private: /** @var The high threshold in bytes for reclamation to be triggered (per-thread) */ static constexpr const char* DEFAULT_GC_HIGH_RECLAIM_THRESHOLD = "8 MB"; - static constexpr uint32_t DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 8 * MEGA_BYTE; // 8 MB - static constexpr uint32_t MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 1 * MEGA_BYTE; // 1 MB - static constexpr uint32_t MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB + static constexpr uint64_t DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 8 * MEGA_BYTE; // 8 MB + static constexpr uint64_t MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 1 * MEGA_BYTE; // 1 MB + static constexpr uint64_t MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB /** ------------------ Default JIT Configuration ------------ */ /** @var Default enable JIT compilation and execution. */ @@ -697,7 +697,7 @@ private: static constexpr bool DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION = false; /** @var The default total memory reference used for calculating memory percent value. */ - static constexpr uint32_t DEFAULT_TOTAL_MEMORY_MB = 10 * KILO_BYTE; // 10 MB + static constexpr uint64_t DEFAULT_TOTAL_MEMORY_MB = 10 * KILO_BYTE; // 10 MB /** @brief Loads configuration from main configuration. */ void LoadConfig(); @@ -713,8 +713,7 @@ private: static void UpdateStringConfigItem(std::string& oldValue, const char* newValue, const char* name); template - void UpdateIntConfigItem( - uint64_t& oldValue, T newValue, const char* name, uint64_t lowerBound = 0, uint64_t upperBound = UINT64_MAX) + void UpdateIntConfigItem(uint64_t& oldValue, T newValue, const char* name, uint64_t lowerBound, uint64_t upperBound) { if ((newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) { if (!m_suppressLog) { @@ -733,8 +732,7 @@ private: } template - void UpdateIntConfigItem( - uint32_t& oldValue, T newValue, const char* name, uint32_t lowerBound = 0, uint32_t upperBound = UINT32_MAX) + void UpdateIntConfigItem(uint32_t& oldValue, T newValue, const char* name, uint32_t lowerBound, uint32_t upperBound) { if ((newValue > UINT32_MAX) || (newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) { if (!m_suppressLog) { @@ -752,8 +750,7 @@ private: } template - void UpdateIntConfigItem( - uint16_t& oldValue, T newValue, const char* name, uint16_t lowerBound = 0, uint16_t upperBound = UINT16_MAX) + void UpdateIntConfigItem(uint16_t& oldValue, T newValue, const char* name, uint16_t lowerBound, uint16_t upperBound) { if ((newValue > UINT16_MAX) || (newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) { if (!m_suppressLog) { @@ -784,6 +781,12 @@ private: } } + void UpdateMemConfigItem(uint64_t& oldValue, const char* name, const char* strDefaultValue, uint64_t scale, + uint64_t lowerBound, uint64_t upperBound, bool allowPercentage); + + void UpdateTimeConfigItem(uint64_t& oldValue, const char* name, const char* strDefaultValue, uint64_t scale, + uint64_t lowerBound, uint64_t upperBound); + void UpdateComponentLogLevel(); static int ParseMemoryPercent(const char* memoryValue); diff --git a/src/gausskernel/storage/mot/core/src/system/mot_engine.cpp b/src/gausskernel/storage/mot/core/src/system/mot_engine.cpp index a8534d54bb46d588eab17dcf6e205af4a6239331..1cdd515f94e349ab195995ff11fda67d900eccdc 100644 --- a/src/gausskernel/storage/mot/core/src/system/mot_engine.cpp +++ b/src/gausskernel/storage/mot/core/src/system/mot_engine.cpp @@ -941,7 +941,7 @@ uint64_t MOTEngine::GetCurrentMemoryConsumptionBytes() uint64_t MOTEngine::GetHardMemoryLimitBytes() { - return ((uint64_t)g_memGlobalCfg.m_maxGlobalMemoryMb * MEGA_BYTE); + return g_memGlobalCfg.m_maxGlobalMemoryMb * MEGA_BYTE; } bool MOTEngine::CheckPolicies() diff --git a/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp b/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp index c0361e896f09d2536d51c4c95650a12a7ef2f40a..3e5a112df05d0c89addb9147143711e81a7afa40 100644 --- a/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp +++ b/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp @@ -326,12 +326,12 @@ private: // get max/min global memory and large allocation store for sessions from configuration MOT::MOTConfiguration& motCfg = MOT::GetGlobalConfiguration(); - uint32_t globalMemoryMb = motCfg.m_globalMemoryMaxLimitMB; - uint32_t localMemoryMb = motCfg.m_localMemoryMaxLimitMB; - uint32_t sessionLargeStoreMb = motCfg.m_sessionLargeBufferStoreSizeMB; + uint64_t globalMemoryMb = motCfg.m_globalMemoryMaxLimitMB; + uint64_t localMemoryMb = motCfg.m_localMemoryMaxLimitMB; + uint64_t sessionLargeStoreMb = motCfg.m_sessionLargeBufferStoreSizeMB; // compare the sum with max_process_memory and system total - uint32_t maxReserveMemoryMb = globalMemoryMb + localMemoryMb + sessionLargeStoreMb; + uint64_t maxReserveMemoryMb = globalMemoryMb + localMemoryMb + sessionLargeStoreMb; // if the total memory is less than the required minimum, then issue a warning, fix it and return if (maxReserveMemoryMb < MOT_MIN_MEMORY_USAGE_MB) { @@ -342,24 +342,25 @@ private: } // get system total - uint32_t systemTotalMemoryMb = MOT::GetTotalSystemMemoryMb(); + uint64_t systemTotalMemoryMb = MOT::GetTotalSystemMemoryMb(); if (systemTotalMemoryMb == 0) { MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, "Load Configuration", "Cannot retrieve total system memory"); return false; } // get envelope limit - uint32_t processTotalMemoryMb = g_instance.attr.attr_memory.max_process_memory / KILO_BYTE; + uint64_t processTotalMemoryMb = g_instance.attr.attr_memory.max_process_memory / KILO_BYTE; // compute the real limit - uint32_t upperLimitMb = min(systemTotalMemoryMb, processTotalMemoryMb); + uint64_t upperLimitMb = min(systemTotalMemoryMb, processTotalMemoryMb); // get dynamic gap we need to preserve between MOT and envelope - uint32_t dynamicGapMb = MIN_DYNAMIC_PROCESS_MEMORY / KILO_BYTE; + uint64_t dynamicGapMb = MIN_DYNAMIC_PROCESS_MEMORY / KILO_BYTE; - MOT_LOG_TRACE("Checking for memory limits: globalMemoryMb=%u, localMemoryMb=%u, sessionLargeStoreMb=%u, " - "systemTotalMemoryMb=%u, processTotalMemoryMb=%u, upperLimitMb=%u, dynamicGapMb=%u, " - "max_process_memory=%u", + MOT_LOG_TRACE("Checking for memory limits: globalMemoryMb=%" PRIu64 ", localMemoryMb=%" PRIu64 + ", sessionLargeStoreMb=%" PRIu64 ", systemTotalMemoryMb=%" PRIu64 + ", processTotalMemoryMb=%" PRIu64 ", upperLimitMb=%" PRIu64 ", dynamicGapMb=%" PRIu64 + ", max_process_memory=%u", globalMemoryMb, localMemoryMb, sessionLargeStoreMb, @@ -372,11 +373,12 @@ private: // we check that a 2GB gap is preserved if (upperLimitMb < maxReserveMemoryMb + dynamicGapMb) { // memory restriction conflict, issue warning and adjust values - MOT_LOG_TRACE( - "MOT engine maximum memory definitions (global: %u MB, local: %u MB, session large store: %u MB, " - "total: " - "%u MB) breach GaussDB maximum process memory restriction (%u MB) and/or total system memory (%u MB). " - "MOT values shall be adjusted accordingly to preserve required gap (%u MB).", + MOT_LOG_TRACE("MOT engine maximum memory definitions (global: %" PRIu64 " MB, local: %" PRIu64 + " MB, session large " + "store: %" PRIu64 " MB, total: %" PRIu64 + " MB) breach GaussDB maximum process memory restriction (%" PRIu64 + " MB) and/or total system memory (%" PRIu64 " MB). " + "MOT values shall be adjusted accordingly to preserve required gap (%" PRIu64 " MB).", globalMemoryMb, localMemoryMb, sessionLargeStoreMb, @@ -386,7 +388,7 @@ private: dynamicGapMb); // compute new total memory limit for MOT - uint32_t newTotalMemoryMb = 0; + uint64_t newTotalMemoryMb = 0; if (upperLimitMb < dynamicGapMb) { // this can happen only if system memory is less than 2GB, we still allow minor breach MOT_LOG_WARN("Using minimal memory limits in MOT Engine due to system total memory restrictions"); @@ -411,20 +413,20 @@ private: return result; } - bool ConfigureMemoryLimits(uint32_t newTotalMemoryMb, uint32_t currentTotalMemoryMb, uint32_t currentGlobalMemoryMb, - uint32_t currentLocalMemoryMb) + bool ConfigureMemoryLimits(uint64_t newTotalMemoryMb, uint64_t currentTotalMemoryMb, uint64_t currentGlobalMemoryMb, + uint64_t currentLocalMemoryMb) { - uint32_t newGlobalMemoryMb = 0; - uint32_t newLocalMemoryMb = 0; - uint32_t newSessionLargeStoreMemoryMb = 0; + uint64_t newGlobalMemoryMb = 0; + uint64_t newLocalMemoryMb = 0; + uint64_t newSessionLargeStoreMemoryMb = 0; // compute new configuration values if (currentTotalMemoryMb > 0) { // we preserve the existing ratio between global and local memory, but reduce the total sum as required double ratio = ((double)newTotalMemoryMb) / ((double)currentTotalMemoryMb); - newGlobalMemoryMb = (uint32_t)(currentGlobalMemoryMb * ratio); + newGlobalMemoryMb = (uint64_t)(currentGlobalMemoryMb * ratio); if (MOT::GetGlobalConfiguration().m_sessionLargeBufferStoreSizeMB > 0) { - newLocalMemoryMb = (uint32_t)(currentLocalMemoryMb * ratio); + newLocalMemoryMb = (uint64_t)(currentLocalMemoryMb * ratio); newSessionLargeStoreMemoryMb = newTotalMemoryMb - newGlobalMemoryMb - newLocalMemoryMb; } else { // if the user configured zero for the session large store, then we want to keep it this way @@ -433,13 +435,13 @@ private: } } else { // when current total memory is zero we split the new total between global and local in ratio of 4:1 - newGlobalMemoryMb = (uint32_t)(newTotalMemoryMb * 0.8f); // 80% to global memory + newGlobalMemoryMb = (uint64_t)(newTotalMemoryMb * 0.8f); // 80% to global memory newLocalMemoryMb = newTotalMemoryMb - newGlobalMemoryMb; // 20% to local memory // session large store remains zero! } - MOT_LOG_WARN( - "Adjusting MOT memory limits: global = %u MB, local = %u MB, session large store = %u MB, total = %u MB", + MOT_LOG_WARN("Adjusting MOT memory limits: global = %" PRIu64 " MB, local = %" PRIu64 + " MB, session large store = %" PRIu64 " MB, total = %" PRIu64 " MB", newGlobalMemoryMb, newLocalMemoryMb, newSessionLargeStoreMemoryMb, @@ -447,14 +449,14 @@ private: // stream into MOT new definitions MOT::mot_string memCfg; - memCfg.format("%u MB", newGlobalMemoryMb); + memCfg.format("%" PRIu64 " MB", newGlobalMemoryMb); bool result = AddExtStringConfigItem("", "max_mot_global_memory", memCfg.c_str()); if (result) { - memCfg.format("%u MB", newLocalMemoryMb); + memCfg.format("%" PRIu64 " MB", newLocalMemoryMb); result = AddExtStringConfigItem("", "max_mot_local_memory", memCfg.c_str()); } if (result) { - memCfg.format("%u MB", newSessionLargeStoreMemoryMb); + memCfg.format("%" PRIu64 " MB", newSessionLargeStoreMemoryMb); result = AddExtStringConfigItem("", "session_large_buffer_store_size", memCfg.c_str()); } @@ -740,9 +742,9 @@ void MOTAdaptor::Init() } // check max process memory here - we do it anyway to protect ourselves from miscalculations - uint32_t globalMemoryKb = MOT::g_memGlobalCfg.m_maxGlobalMemoryMb * KILO_BYTE; - uint32_t localMemoryKb = MOT::g_memGlobalCfg.m_maxLocalMemoryMb * KILO_BYTE; - uint32_t maxReserveMemoryKb = globalMemoryKb + localMemoryKb; + uint64_t globalMemoryKb = MOT::g_memGlobalCfg.m_maxGlobalMemoryMb * KILO_BYTE; + uint64_t localMemoryKb = MOT::g_memGlobalCfg.m_maxLocalMemoryMb * KILO_BYTE; + uint64_t maxReserveMemoryKb = globalMemoryKb + localMemoryKb; if ((g_instance.attr.attr_memory.max_process_memory < (int32)maxReserveMemoryKb) || ((g_instance.attr.attr_memory.max_process_memory - maxReserveMemoryKb) < MIN_DYNAMIC_PROCESS_MEMORY)) { @@ -757,8 +759,8 @@ void MOTAdaptor::Init() MOT::MOTEngine::DestroyInstance(); elog(FATAL, "The value of pre-reserved memory for MOT engine is not reasonable: " - "Request for a maximum of %u KB global memory, and %u KB session memory (total of %u KB) " - "is invalid since max_process_memory is %u KB", + "Request for a maximum of %" PRIu64 " KB global memory, and %" PRIu64 + " KB session memory (total of %" PRIu64 " KB) is invalid since max_process_memory is %u KB", globalMemoryKb, localMemoryKb, maxReserveMemoryKb,