From fb617c6676cd57f4c7943df73239d968d7c26488 Mon Sep 17 00:00:00 2001 From: Vinoth Date: Thu, 13 Aug 2020 11:45:53 +0800 Subject: [PATCH] MOT fixes in gs_dump and gs_ctl --- src/bin/pg_ctl/fetchmot.cpp | 11 +++- src/bin/pg_dump/pg_dump.cpp | 24 +++++--- .../optimizer/commands/tablecmds.cpp | 2 +- .../mot/core/src/memory/sys_numa_api.cpp | 4 +- .../storage/mot/core/src/storage/table.cpp | 61 +++++++++++++++---- .../storage/mot/core/src/storage/table.h | 20 +++++- .../mot/core/src/system/transaction/txn.cpp | 55 +++++++++++++---- .../storage/mot/fdw_adapter/src/mot_fdw.cpp | 5 +- .../mot/fdw_adapter/src/mot_internal.cpp | 29 +++++++-- 9 files changed, 163 insertions(+), 48 deletions(-) diff --git a/src/bin/pg_ctl/fetchmot.cpp b/src/bin/pg_ctl/fetchmot.cpp index b5c04f6fcc..a518b03869 100644 --- a/src/bin/pg_ctl/fetchmot.cpp +++ b/src/bin/pg_ctl/fetchmot.cpp @@ -45,9 +45,9 @@ static uint64 totaldone = 0; * the contents of it into a directory. Only files, directories and * symlinks are supported, no other kinds of special files. */ -static void MotReceiveAndUnpackTarFile( - const char* basedir, const char* chkptName, PGconn* conn, PGresult* res, int rownum, const char* progname) +static void MotReceiveAndUnpackTarFile(const char* basedir, const char* chkptName, PGconn* conn, const char* progname) { + PGresult* res = NULL; char current_path[MAXPGPATH]; char filename[MAXPGPATH]; int current_len_left; @@ -66,6 +66,7 @@ static void MotReceiveAndUnpackTarFile( fprintf(stderr, "%s: could not get COPY data stream: %s", progname, PQerrorMessage(conn)); disconnect_and_exit(1); } + PQclear(res); while (1) { int r; @@ -302,6 +303,7 @@ void FetchMotCheckpoint(const char* basedir, PGconn* fetchConn, const char* prog if (verbose) { fprintf(stderr, "%s: no mot checkpoint exists\n", progname); } + PQclear(res); return; } else { fprintf(stderr, "%s: could not fetch mot checkpoint info: %s", progname, PQerrorMessage(fetchConn)); @@ -341,7 +343,7 @@ void FetchMotCheckpoint(const char* basedir, PGconn* fetchConn, const char* prog exit(1); } - MotReceiveAndUnpackTarFile(basedir, chkptName, fetchConn, res, 1, progname); + MotReceiveAndUnpackTarFile(basedir, chkptName, fetchConn, progname); if (verbose) { fprintf(stderr, "%s: finished fetching mot checkpoint\n", progname); } @@ -400,6 +402,9 @@ char* GetOptionValueFromFile(const char* fileName, const char* option) } } + if (line != NULL) { + free(line); + } fclose(file); return ret; } diff --git a/src/bin/pg_dump/pg_dump.cpp b/src/bin/pg_dump/pg_dump.cpp index aa7efceffe..121d546e78 100644 --- a/src/bin/pg_dump/pg_dump.cpp +++ b/src/bin/pg_dump/pg_dump.cpp @@ -17390,12 +17390,14 @@ static void dumpConstraintForForeignTbl(Archive* fout, ConstraintInfo* coninfo) appendPQExpBuffer(q, ";\n"); /* - * DROP must be fully qualified in case same name appears in - * pg_catalog + * DROP must be fully qualified in case same name appears in pg_catalog + * In case of FDW MOT do not generate drop statements for CONSTRAINT */ - appendPQExpBuffer(delq, "ALTER TABLE %s.", fmtId(tbinfo->dobj.nmspace->dobj.name)); - appendPQExpBuffer(delq, "%s ", fmtId(tbinfo->dobj.name)); - appendPQExpBuffer(delq, "DROP CONSTRAINT %s;\n", fmtId(coninfo->dobj.name)); + if (!tbinfo->isMOT) { + appendPQExpBuffer(delq, "ALTER TABLE %s.", fmtId(tbinfo->dobj.nmspace->dobj.name)); + appendPQExpBuffer(delq, "%s ", fmtId(tbinfo->dobj.name)); + appendPQExpBuffer(delq, "DROP CONSTRAINT %s;\n", fmtId(coninfo->dobj.name)); + } ArchiveEntry(fout, coninfo->dobj.catId, coninfo->dobj.dumpId, @@ -18972,7 +18974,12 @@ static void addBoundaryDependencies(DumpableObject** dobjs, int numObjs, Dumpabl case DO_DEFAULT_ACL: case DO_RLSPOLICY: /* Post-data objects: must come after the post-data boundary */ - addObjectDependency(dobj, postDataBound->dumpId); + if (dobj->objType == DO_INDEX && + ((IndxInfo*)dobj)->indextable && ((IndxInfo*)dobj)->indextable->isMOT) { + addObjectDependency(preDataBound, dobj->dumpId); + } else { + addObjectDependency(dobj, postDataBound->dumpId); + } break; case DO_RULE: /* Rules are post-data, but only if dumped separately */ @@ -18983,8 +18990,11 @@ static void addBoundaryDependencies(DumpableObject** dobjs, int numObjs, Dumpabl case DO_FK_CONSTRAINT: case DO_FTBL_CONSTRAINT: /* Constraints are post-data, but only if dumped separately */ - if (((ConstraintInfo*)dobj)->separate) + if (((ConstraintInfo*)dobj)->contable && ((ConstraintInfo*)dobj)->contable->isMOT) { + addObjectDependency(preDataBound, dobj->dumpId); + } else if (((ConstraintInfo*)dobj)->separate) { addObjectDependency(dobj, postDataBound->dumpId); + } break; case DO_PRE_DATA_BOUNDARY: /* nothing to do */ diff --git a/src/gausskernel/optimizer/commands/tablecmds.cpp b/src/gausskernel/optimizer/commands/tablecmds.cpp index e545dfcb92..bb19d3b2a0 100644 --- a/src/gausskernel/optimizer/commands/tablecmds.cpp +++ b/src/gausskernel/optimizer/commands/tablecmds.cpp @@ -5370,7 +5370,6 @@ void AlterTable(Oid relid, LOCKMODE lockmode, AlterTableStmt* stmt) * wheather support alter owner, support alter colomn type, etc. */ FdwRoutine* fdwroutine = GetFdwRoutineByRelId(relid); - stmt->relation->foreignOid = relid; if (NULL != fdwroutine->ValidateTableDef) { fdwroutine->ValidateTableDef((Node*)stmt); @@ -8842,6 +8841,7 @@ static void ATExecAddIndex(AlteredTableInfo* tab, Relation rel, IndexStmt* stmt, check_rights, skip_build, quiet); + CreateForeignIndex(stmt, new_index); (void)pgstat_report_waitstatus(oldStatus); /* diff --git a/src/gausskernel/storage/mot/core/src/memory/sys_numa_api.cpp b/src/gausskernel/storage/mot/core/src/memory/sys_numa_api.cpp index 00c51220cc..b4b63159eb 100644 --- a/src/gausskernel/storage/mot/core/src/memory/sys_numa_api.cpp +++ b/src/gausskernel/storage/mot/core/src/memory/sys_numa_api.cpp @@ -564,7 +564,9 @@ static void MotSysNumaSetTaskConstraints() g_numProcNode = MotSysNumaReadBitmask(mask, g_allNodesBm); } fclose(f); - free(buffer); + if (buffer != nullptr) { + free(buffer); + } if (g_numProcCpu <= 0) { for (i = 0; i <= g_maxConfCpu; i++) { diff --git a/src/gausskernel/storage/mot/core/src/storage/table.cpp b/src/gausskernel/storage/mot/core/src/storage/table.cpp index 7e3038e701..17f8255692 100644 --- a/src/gausskernel/storage/mot/core/src/storage/table.cpp +++ b/src/gausskernel/storage/mot/core/src/storage/table.cpp @@ -157,27 +157,60 @@ void Table::DecIndexColumnUsage(Index* index) } } -void Table::AddPrimaryIndex(Index*&& index) +bool Table::IsTableEmpty(uint32_t tid) { - index->SetTable(this); - this->m_primaryIndex = std::move(index); - IncIndexColumnUsage(index); + bool res = false; + IndexIterator* it = GetPrimaryIndex()->Begin(tid); + + // report error if failed to allocate + if (it == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "IsTableEmpty", "Failed to begin iterating over primary index"); + return false; + } + if (!it->IsValid()) { + res = true; + } + + delete it; + return res; +} + +void Table::SetPrimaryIndex(Index* index) +{ + if (index != nullptr) { + index->SetTable(this); + } + this->m_primaryIndex = index; m_indexes[0] = index; - ++m_numIndexes; } -void Table::UpdatePrimaryIndex(Index*&& index) +bool Table::UpdatePrimaryIndex(Index* index, TxnManager* txn, uint32_t tid) { if (this->m_primaryIndex) { - DecIndexColumnUsage(this->m_primaryIndex); - delete this->m_primaryIndex; + if (txn == nullptr) { + DeletePrimaryIndex(this->m_primaryIndex); + } else { + txn->DropIndex(this->m_primaryIndex); + } + } else { + if (m_numIndexes == 0) { + ++m_numIndexes; + } } - index->SetTable(this); - this->m_primaryIndex = std::move(index); + IncIndexColumnUsage(index); + SetPrimaryIndex(index); - m_indexes[0] = index; + return true; +} + +RC Table::DeletePrimaryIndex(MOT::Index* index) +{ + DecIndexColumnUsage(index); + delete index; + + return RC::RC_OK; } bool Table::AddSecondaryIndex(const string& indexName, Index* index, TxnManager* txn, uint32_t tid) @@ -913,7 +946,11 @@ RC Table::CreateIndexFromMeta(CommonIndexMeta& meta, bool primary, uint32_t tid) return RC_ERROR; } if (primary) { - UpdatePrimaryIndex((Index*)ix); + if (UpdatePrimaryIndex(ix, nullptr, tid) != true) { + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, "Create Index from meta-data", "Failed to add primary index"); + delete ix; + return RC_ERROR; + } } else { if (AddSecondaryIndex(ix->GetName(), (Index*)ix, nullptr, tid) != true) { MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, "Create Index from meta-data", "Failed to add secondary index"); diff --git a/src/gausskernel/storage/mot/core/src/storage/table.h b/src/gausskernel/storage/mot/core/src/storage/table.h index ed0d83f259..65647a553c 100644 --- a/src/gausskernel/storage/mot/core/src/storage/table.h +++ b/src/gausskernel/storage/mot/core/src/storage/table.h @@ -182,13 +182,15 @@ public: * @brief Sets the primary index for the table. * @param index The index to set. */ - void AddPrimaryIndex(Index*&& index); + void SetPrimaryIndex(Index* index); /** * @brief Sets the primary index for the table (replaces previously created fake primary. * @param index The index to set. + * @param txn The current transaction. + * @param tid Current thread id */ - void UpdatePrimaryIndex(Index*&& index); + bool UpdatePrimaryIndex(Index* index, TxnManager* txn, uint32_t tid); /** * @brief Adds a secondary index to the table. @@ -216,6 +218,20 @@ public: */ RC RemoveSecondaryIndex(char* name, TxnManager* txn); + /** + * @brief Deletes a primary index. + * @param the index to remove. + * @return RC value denoting the operation's completion status. + */ + RC DeletePrimaryIndex(MOT::Index* index); + + /** + * @brief Checks if table contains data. + * @param thread id. + * @return true if table is empty, false otherwise. + */ + bool IsTableEmpty(uint32_t tid); + /** * @brief Truncates the table. * @param txn The txn manager object. diff --git a/src/gausskernel/storage/mot/core/src/system/transaction/txn.cpp b/src/gausskernel/storage/mot/core/src/system/transaction/txn.cpp index 62ec7849da..9ba9671804 100644 --- a/src/gausskernel/storage/mot/core/src/system/transaction/txn.cpp +++ b/src/gausskernel/storage/mot/core/src/system/transaction/txn.cpp @@ -534,7 +534,13 @@ void TxnManager::RollbackDDLs() MOT_LOG_INFO("Rollback of create index %s for table %s", index->GetName().c_str(), table->GetLongTableName().c_str()); - table->RemoveSecondaryIndex((char*)index->GetName().c_str(), this); + if (index->IsPrimaryKey()) { + table->SetPrimaryIndex(nullptr); + GcManager::ClearIndexElements(index->GetIndexId()); + table->DeletePrimaryIndex(index); + } else { + table->RemoveSecondaryIndex((char*)index->GetName().c_str(), this); + } break; case DDL_ACCESS_DROP_INDEX: index = (Index*)ddl_access->GetEntry(); @@ -542,6 +548,9 @@ void TxnManager::RollbackDDLs() MOT_LOG_INFO("Rollback of drop index %s for table %s", index->GetName().c_str(), table->GetLongTableName().c_str()); + if (index->IsPrimaryKey()) { + table->SetPrimaryIndex(index); + } break; default: break; @@ -582,15 +591,24 @@ void TxnManager::WriteDDLChanges() delete[] indexes; break; case DDL_ACCESS_CREATE_INDEX: - ((Index*)ddl_access->GetEntry())->SetIsCommited(true); + index = (Index*)ddl_access->GetEntry(); + table = index->GetTable(); + index->SetIsCommited(true); + if (index->IsPrimaryKey()) { + table->SetPrimaryIndex(index); + } break; case DDL_ACCESS_DROP_INDEX: index = (Index*)ddl_access->GetEntry(); - if (index->IsPrimaryKey()) - break; table = index->GetTable(); table->WrLock(); - table->RemoveSecondaryIndex((char*)index->GetName().c_str(), this); + if (index->IsPrimaryKey()) { + table->SetPrimaryIndex(nullptr); + GcManager::ClearIndexElements(index->GetIndexId()); + table->DeletePrimaryIndex(index); + } else { + table->RemoveSecondaryIndex((char*)index->GetName().c_str(), this); + } table->Unlock(); break; default: @@ -1225,14 +1243,22 @@ RC TxnManager::CreateIndex(Table* table, Index* index, bool is_primary) return RC_MEMORY_ALLOCATION_ERROR; } + table->WrLock(); // for concurrent access if (is_primary) { - table->UpdatePrimaryIndex((MOT::Index*)index); + if (!table->UpdatePrimaryIndex(index, this, m_threadId)) { + table->Unlock(); + if (MOT_IS_OOM()) { // do not report error in "unique violation" scenario + MOT_REPORT_ERROR( + MOT_ERROR_INTERNAL, "Create Index", "Failed to add primary index %s", index->GetName().c_str()); + } + delete ddl_access; + return m_err; + } } else { // currently we are still adding the index to the table although // is should only be added on successful commit. Assuming that if // a client did a create index, all other clients are waiting on a lock // until the changes are either commited or aborted - table->WrLock(); // for concurrent access if (table->GetNumIndexes() == MAX_NUM_INDEXES) { table->Unlock(); MOT_REPORT_ERROR(MOT_ERROR_RESOURCE_LIMIT, @@ -1253,14 +1279,13 @@ RC TxnManager::CreateIndex(Table* table, Index* index, bool is_primary) delete ddl_access; return m_err; } - table->Unlock(); } - + table->Unlock(); m_txnDdlAccess->Add(ddl_access); return RC_OK; } -RC TxnManager::DropIndex(Index* index) +RC TxnManager::DropIndex(MOT::Index* index) { // allocate DDL before work and fail immediately if required TxnDDLAccess::DDLAccess* new_ddl_access = nullptr; @@ -1277,13 +1302,19 @@ RC TxnManager::DropIndex(Index* index) RC res = RC_OK; Table* table = index->GetTable(); - RollbackSecondaryIndexInsert(index); + if (!index->IsPrimaryKey()) { + RollbackSecondaryIndexInsert(index); + } if (ddl_access != nullptr) { // this index was created in this transaction, can delete it from the ddl_access // table->removeSecondaryIndex also performs releases the object m_txnDdlAccess->EraseByOid(index->GetExtId()); - res = table->RemoveSecondaryIndex((char*)index->GetName().c_str(), this); + if (index->IsPrimaryKey()) { + res = table->DeletePrimaryIndex(index); + } else { + res = table->RemoveSecondaryIndex((char*)index->GetName().c_str(), this); + } if (res != RC_OK) { // print Error MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, "Drop Index", "Failed to remove secondary index"); diff --git a/src/gausskernel/storage/mot/fdw_adapter/src/mot_fdw.cpp b/src/gausskernel/storage/mot/fdw_adapter/src/mot_fdw.cpp index 008cacfaf6..bdbc208d5c 100644 --- a/src/gausskernel/storage/mot/fdw_adapter/src/mot_fdw.cpp +++ b/src/gausskernel/storage/mot/fdw_adapter/src/mot_fdw.cpp @@ -1813,11 +1813,8 @@ static void MOTValidateTableDef(Node* obj) cmd = (AlterTableCmd*)lfirst(cell); if (cmd->subtype == AT_ChangeOwner) { allow = true; - } else if (cmd->subtype == AT_AddIndex && u_sess->proc_cxt.clientIsGsRestore) { + } else if (cmd->subtype == AT_AddIndex) { allow = true; - isMemoryLimitReached(); - ((IndexStmt*)cmd->def)->relation->foreignOid = ats->relation->foreignOid; - MOTAdaptor::CreateIndex((IndexStmt*)cmd->def, tid); } } if (allow == false) { diff --git a/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp b/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp index 408101f9b0..f1dd1f1eab 100644 --- a/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp +++ b/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp @@ -1541,7 +1541,16 @@ MOT::RC MOTAdaptor::CreateIndex(IndexStmt* index, ::TransactionId tid) return MOT::RC_ERROR; } - if (table->GetNumIndexes() == MAX_NUM_INDEXES) { + if (index->primary) { + if (!table->IsTableEmpty(txn->GetThdId())) { + ereport(ERROR, + (errmodule(MOD_MM), + errcode(ERRCODE_FDW_ERROR), + errmsg( + "Table %s is not empty, create primary index is not allowed", table->GetTableName().c_str()))); + return MOT::RC_ERROR; + } + } else if (table->GetNumIndexes() == MAX_NUM_INDEXES) { ereport(ERROR, (errmodule(MOD_MM), errcode(ERRCODE_FDW_TOO_MANY_INDEXES), @@ -1852,6 +1861,15 @@ MOT::RC MOTAdaptor::CreateTable(CreateForeignTableStmt* table, ::TransactionId t table->base.relation->foreignOid, columnCount, tupleSize); + + res = txn->CreateTable(currentTable); + if (res != MOT::RC_OK) { + delete currentTable; + currentTable = nullptr; + report_pg_error(res, txn); + break; + } + // add default PK index MOT::RC rc = MOT::RC_OK; primaryIdx = MOT::IndexFactory::CreatePrimaryIndexEx(MOT::IndexingMethod::INDEXING_METHOD_TREE, @@ -1866,20 +1884,19 @@ MOT::RC MOTAdaptor::CreateTable(CreateForeignTableStmt* table, ::TransactionId t report_pg_error(rc, txn); break; } + primaryIdx->SetExtId(table->base.relation->foreignOid + 1); primaryIdx->SetNumTableFields(columnCount); primaryIdx->SetNumIndexFields(1); primaryIdx->SetLenghtKeyFields(0, -1, 8); primaryIdx->SetFakePrimary(true); - // The primary index has been created and registered at the table, but not yet initialized - currentTable->AddPrimaryIndex(std::move(primaryIdx)); - - res = txn->CreateTable(currentTable); + // Add default primary index + res = txn->CreateIndex(currentTable, primaryIdx, true); } while (0); if (res != MOT::RC_OK) { if (currentTable != nullptr) { - MOT::GetTableManager()->DropTable(currentTable, txn->GetSessionContext()); + txn->DropTable(currentTable); } if (primaryIdx != nullptr) { delete primaryIdx; -- Gitee