diff --git a/src/bin/pg_basebackup/pg_basebackup.cpp b/src/bin/pg_basebackup/pg_basebackup.cpp index 2bb16734dcead10799321469c383a2fa2a6172df..52217cf0939a902d080d6f103b185fd3a051c3d2 100644 --- a/src/bin/pg_basebackup/pg_basebackup.cpp +++ b/src/bin/pg_basebackup/pg_basebackup.cpp @@ -19,7 +19,7 @@ /* * We have to use postgres.h not postgres_fe.h here, because there's so much * backend-only stuff in the XLOG include files we need. But we need a - * frontend-ish environment otherwise. Hence this ugly hack. + * frontend-ish environment otherwise. Hence this ugly hack. */ #include "postgres.h" #include "knl/knl_variable.h" @@ -31,13 +31,10 @@ #include #include -#ifdef HAVE_LIBZ -#include "zlib.h" -#endif - #include "getopt_long.h" #include "receivelog.h" #include "streamutil.h" +#include "gs_tar_const.h" #include "bin/elog.h" #include "lib/string.h" @@ -55,7 +52,6 @@ typedef struct TablespaceList { /* Global options */ char* basedir = NULL; static TablespaceList tablespacee_dirs = {NULL, NULL}; -static const int BLOCK_SIZE = 2560; /* gs_tar block size */ char* g_xlogOption = NULL; char format = 'p'; /* p(lain)/t(ar) */ char* label = "gs_basebackup base backup"; @@ -66,17 +62,6 @@ bool includewal = false; bool streamwal = false; bool fastcheckpoint = false; -/* fileStream result */ -static const int READ_ERROR = -2; - -/* gs_tar offset */ -static const int LEN_LEFT = 1048; -static const int FILE_PADDING = 511; /* All files are padded up to 512 bytes */ -static const int FILE_TYPE = 1080; - -/* file type */ -static const char TYPE_DICTORY = '5'; - extern char** tblspaceDirectory; extern int tblspaceCount; extern int tblspaceIndex; @@ -122,21 +107,8 @@ static int GsTar(int argc, char** argv); static int GsBaseBackup(int argc, char** argv); static const char* get_tablespace_mapping(const char* dir); -extern void FetchMotCheckpoint(const char* basedir, PGconn* fetchConn, const char* progname, bool verbose); - -#ifdef HAVE_LIBZ -static const char* get_gz_error(gzFile gzf) -{ - int errnum; - const char* errmsg = NULL; - - errmsg = gzerror(gzf, &errnum); - if (errnum == Z_ERRNO) - return strerror(errno); - else - return errmsg; -} -#endif +extern void FetchMotCheckpoint(const char* basedir, PGconn* fetchConn, const char* progname, bool verbose, + const char format = 'p', const int compresslevel = 0); /* * Split argument into old_dir and new_dir and append to tablespace mapping @@ -144,7 +116,7 @@ static const char* get_gz_error(gzFile gzf) */ static void tablespace_list_append(const char* arg) { - TablespaceListCell* cell = (TablespaceListCell*)malloc(sizeof(TablespaceListCell)); + TablespaceListCell* cell = (TablespaceListCell*)xmalloc0(sizeof(TablespaceListCell)); char* dst = NULL; char* dst_ptr = NULL; const char* arg_ptr = NULL; @@ -601,32 +573,6 @@ static void progress_report(int tablespacenum, const char* filename) fprintf(stderr, "\r"); } -#ifdef HAVE_LIBZ -static gzFile openGzFile(const char* filename) -{ - gzFile ztarfile = gzopen(filename, "wb"); - if (gzsetparams(ztarfile, compresslevel, Z_DEFAULT_STRATEGY) != Z_OK) { - fprintf( - stderr, _("%s: could not set compression level %d: %s\n"), progname, compresslevel, get_gz_error(ztarfile)); - disconnect_and_exit(1); - } - return ztarfile; -} - -static void writeGzFile(gzFile ztarfile, char* copybuf, int buf_size, char* filename) -{ - if (gzwrite(ztarfile, copybuf, buf_size) != buf_size) { - fprintf(stderr, - _("%s: could not write to compressed file \"%s\": %s\n"), - progname, - filename, - get_gz_error(ztarfile)); - disconnect_and_exit(1); - } -} - -#endif - /* * Receive a tar format file from the connection to the server, and write * the data from this file directly into a tar file. If compression is @@ -687,7 +633,15 @@ static void ReceiveTarFile(PGconn* conn, PGresult* res, int rownum) if (compresslevel != 0) { errorno = snprintf_s(filename, sizeof(filename), sizeof(filename) - 1, "%s/base.tar.gz", basedir); securec_check_ss_c(errorno, "", ""); - ztarfile = openGzFile(filename); + ztarfile = openGzFile(filename, compresslevel); + if (ztarfile == NULL) { + fprintf(stderr, + _("%s: could not set compression level %d: %s\n"), + progname, + compresslevel, + get_gz_error(ztarfile)); + disconnect_and_exit(1); + } } else #endif { @@ -706,7 +660,7 @@ static void ReceiveTarFile(PGconn* conn, PGresult* res, int rownum) errorno = snprintf_s( filename, sizeof(filename), sizeof(filename) - 1, "%s/%s.tar.gz", basedir, PQgetvalue(res, rownum, 0)); securec_check_ss_c(errorno, "", ""); - ztarfile = openGzFile(filename); + ztarfile = openGzFile(filename, compresslevel); } else #endif { @@ -760,30 +714,6 @@ static void ReceiveTarFile(PGconn* conn, PGresult* res, int rownum) int r = PQgetCopyData(conn, ©buf, 0); if (r == -1) { - /* - * End of chunk. Close file (but not stdout). - * - * Also, write two completely empty blocks at the end of the tar - * file, as required by some tar programs. - */ - char zerobuf[1024]; - - errorno = memset_s(zerobuf, sizeof(zerobuf), 0, sizeof(zerobuf)); - securec_check_c(errorno, "", ""); -#ifdef HAVE_LIBZ - if (ztarfile != NULL) { - writeGzFile(ztarfile, zerobuf, sizeof(zerobuf), filename); - } else -#endif - { - if (fwrite(zerobuf, sizeof(zerobuf), 1, tarfile) != 1) { - fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); - fclose(tarfile); - tarfile = NULL; - disconnect_and_exit(1); - } - } - #ifdef HAVE_LIBZ if (ztarfile != NULL) { if (gzclose(ztarfile) != 0) { @@ -807,7 +737,6 @@ static void ReceiveTarFile(PGconn* conn, PGresult* res, int rownum) tarfile = NULL; } } - break; } else if (r == -2) { fprintf(stderr, _("%s: could not read COPY data: %s"), progname, PQerrorMessage(conn)); @@ -816,7 +745,14 @@ static void ReceiveTarFile(PGconn* conn, PGresult* res, int rownum) #ifdef HAVE_LIBZ if (ztarfile != NULL) { - writeGzFile(ztarfile, copybuf, r, filename); + if (!writeGzFile(ztarfile, copybuf, r)) { + fprintf(stderr, + _("%s: could not write to compressed file \"%s\": %s\n"), + progname, + filename, + get_gz_error(ztarfile)); + disconnect_and_exit(1); + } } else #endif { @@ -960,7 +896,7 @@ static void ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) } break; - } else if (r == READ_ERROR) { + } else if (r == TAR_READ_ERROR) { fprintf(stderr, _("%s: could not read COPY data: %s"), progname, PQerrorMessage(conn)); disconnect_and_exit(1); } @@ -972,13 +908,13 @@ static void ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) /* * No current file, so this must be the header for a new file */ - if (r != BLOCK_SIZE) { + if (r != TAR_BLOCK_SIZE) { fprintf(stderr, _("%s: invalid tar block header size: %d\n"), progname, r); disconnect_and_exit(1); } - totaldone += BLOCK_SIZE; + totaldone += TAR_BLOCK_SIZE; - if (sscanf_s(copybuf + LEN_LEFT, "%201o", ¤t_len_left) != 1) { + if (sscanf_s(copybuf + TAR_LEN_LEFT, "%201o", ¤t_len_left) != 1) { fprintf(stderr, _("%s: could not parse file size\n"), progname); disconnect_and_exit(1); } @@ -992,7 +928,7 @@ static void ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) /* * All files are padded up to 512 bytes */ - current_padding = ((current_len_left + FILE_PADDING) & ~FILE_PADDING) - current_len_left; + current_padding = ((current_len_left + TAR_FILE_PADDING) & ~TAR_FILE_PADDING) - current_len_left; /* * First part of header is zero terminated filename @@ -1010,7 +946,7 @@ static void ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) /* * Ends in a slash means directory or symlink to directory */ - if (copybuf[FILE_TYPE] == TYPE_DICTORY) { + if (copybuf[TAR_FILE_TYPE] == TAR_TYPE_DICTORY) { /* * Directory */ @@ -1039,12 +975,12 @@ static void ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) filename, strerror(errno)); #endif - } else if (copybuf[FILE_TYPE] == '2') { + } else if (copybuf[TAR_FILE_TYPE] == '2') { /* * Symbolic link */ filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */ - mapped_tblspc_path = get_tablespace_mapping(©buf[FILE_TYPE + 1]); + mapped_tblspc_path = get_tablespace_mapping(©buf[TAR_FILE_TYPE + 1]); if (symlink(mapped_tblspc_path, filename) != 0) { if (IsXlogDir(filename)) { fprintf(stderr, @@ -1072,7 +1008,7 @@ static void ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) sizeof(absolut_path) - 1, "%s/%s", basedir, - ©buf[FILE_TYPE + 1]); + ©buf[TAR_FILE_TYPE + 1]); securec_check_ss_c(errorno, "\0", "\0"); if (symlink(absolut_path, filename) != 0) { @@ -1517,9 +1453,7 @@ static void BaseBackup(void) exit(1); } ClearAndFreePasswd(); - if (format == 'p') { - FetchMotCheckpoint(basedir, conn, progname, (bool)verbose); - } + FetchMotCheckpoint(basedir, conn, progname, (bool)verbose, format, compresslevel); PQfinish(conn); conn = NULL; @@ -1555,7 +1489,7 @@ static void remove_dw_file(const char* dw_file_name, const char* target_dir, cha /* * * * delete existing double write file if existed, recreate it and write one page of zero * * @param target_dir data base root dir - * */ + * */ static void backup_dw_file(const char* target_dir) { int rc; @@ -1708,12 +1642,12 @@ static int GsTar(int argc, char** argv) copybuf = NULL; } - current_len_left = current_len_left == 0 && current_padding == 0 ? BLOCK_SIZE : current_len_left; + current_len_left = current_len_left == 0 && current_padding == 0 ? TAR_BLOCK_SIZE : current_len_left; if (current_len_left == 0 && current_padding != 0) { - copybuf = (char*)malloc(current_padding); + copybuf = (char*)xmalloc0(current_padding); r = fread(copybuf, 1, current_padding, tarfile); } else { - copybuf = (char*)malloc(current_len_left); + copybuf = (char*)xmalloc0(current_len_left); r = fread(copybuf, 1, current_len_left, tarfile); // end of file if ((uint64)r < current_len_left) { @@ -1742,12 +1676,12 @@ static int GsTar(int argc, char** argv) int filemode; /* No current file, so this must be the header for a new file */ - if (r != BLOCK_SIZE) { + if (r != TAR_BLOCK_SIZE) { fprintf(stderr, "%s: invalid tar block header size: %d\n", progname, r); fclose(tarfile); return -1; } - totaldone += BLOCK_SIZE; + totaldone += TAR_BLOCK_SIZE; if (sscanf_s(copybuf + 1048, "%201o", ¤t_len_left) != 1) { fprintf(stderr, "%s: could not parse file size\n", progname); @@ -1858,7 +1792,7 @@ static int GsTar(int argc, char** argv) } } } else { - pg_log(PG_WARNING, "unrecognized link indicator \"%c\"\n", copybuf[FILE_TYPE]); + pg_log(PG_WARNING, "unrecognized link indicator \"%c\"\n", copybuf[TAR_FILE_TYPE]); return -1; } continue; /* directory or link handled */ @@ -1979,7 +1913,7 @@ static int GsBaseBackup(int argc, char** argv) } } - while ((c = getopt_long(argc, argv, "D:l:c:h:p:U:s:X:F:T:wWvPxz", long_options, &option_index)) != -1) { + while ((c = getopt_long(argc, argv, "D:l:c:h:p:U:s:X:F:T:Z:wWvPxz", long_options, &option_index)) != -1) { switch (c) { case 'D': { GS_FREE(basedir); diff --git a/src/bin/pg_ctl/backup.cpp b/src/bin/pg_ctl/backup.cpp index 64afc59f356cc14fe5aa1ba3a1c4b4ee9d45f051..59b7b4aec21c4a9d3e7de959d6ddceb72f04a356 100755 --- a/src/bin/pg_ctl/backup.cpp +++ b/src/bin/pg_ctl/backup.cpp @@ -127,7 +127,8 @@ static int replace_node_name(char* sSrc, const char* sMatchStr, const char* sRep static void show_full_build_process(const char* errmg); static void backup_dw_file(const char* target_dir); static void get_xlog_location(char (&xlog_location)[MAXPGPATH]); -extern void FetchMotCheckpoint(const char* basedir, PGconn* fetchConn, const char* progname, bool verbose); +extern void FetchMotCheckpoint(const char* basedir, PGconn* fetchConn, const char* progname, bool verbose, + const char format = 'p', const int compresslevel = 0); extern char* GetOptionValueFromFile(const char* fileName, const char* option); /* diff --git a/src/bin/pg_ctl/fetchmot.cpp b/src/bin/pg_ctl/fetchmot.cpp index a518b03869b9f9c7fbd8280bcbdb249340edd16c..f70910fccba53407ca0a63886477ba2fc0cf7c4e 100644 --- a/src/bin/pg_ctl/fetchmot.cpp +++ b/src/bin/pg_ctl/fetchmot.cpp @@ -27,6 +27,8 @@ #include #include #include "postgres_fe.h" +#include "gs_tar_const.h" +#include "streamutil.h" #include "libpq/libpq-fe.h" #include "utils/builtins.h" #include "common/fe_memutils.h" @@ -40,6 +42,290 @@ static uint64 totaldone = 0; +static void CheckConnResult(PGconn* conn, const char* progname) +{ + PGresult* res = PQgetResult(conn); + if (PQresultStatus(res) != PGRES_COPY_OUT) { + fprintf(stderr, "%s: could not get COPY data stream: %s", progname, PQerrorMessage(conn)); + disconnect_and_exit(1); + } + PQclear(res); +} + +static void MotReceiveAndAppendTarFile( + const char* basedir, const char* chkptName, PGconn* conn, const char* progname, int compresslevel) +{ + CheckConnResult(conn, progname); + + FILE* tarfile = NULL; + char filename[MAXPGPATH]; + errno_t errorno = EOK; + char* copybuf = NULL; + FILE* file = NULL; + char current_path[MAXPGPATH]; + int current_len_left = 0; + int current_padding = 0; + + errorno = strncpy_s(current_path, sizeof(current_path), basedir, sizeof(current_path) - 1); + securec_check_c(errorno, "", ""); + +#ifdef HAVE_LIBZ + gzFile ztarfile = NULL; + int duplicatedfd = -1; +#endif + if (strcmp(basedir, "-") == 0) { +#ifdef HAVE_LIBZ + if (compresslevel != 0) { + duplicatedfd = dup(fileno(stdout)); + if (duplicatedfd == -1) { + fprintf(stderr, _("%s: could not allocate dup fd by fileno(stdout): %s\n"), progname, strerror(errno)); + disconnect_and_exit(1); + } + + ztarfile = gzdopen(duplicatedfd, "ab"); + if (gzsetparams(ztarfile, compresslevel, Z_DEFAULT_STRATEGY) != Z_OK) { + fprintf(stderr, + _("%s: could not set compression level %d: %s\n"), + progname, + compresslevel, + get_gz_error(ztarfile)); + close(duplicatedfd); + duplicatedfd = -1; + disconnect_and_exit(1); + } + close(duplicatedfd); + duplicatedfd = -1; + } else +#endif + tarfile = stdout; + strcpy_s(filename, 1, "-"); + } else { +#ifdef HAVE_LIBZ + if (compresslevel != 0) { + errorno = snprintf_s(filename, sizeof(filename), sizeof(filename) - 1, "%s/base.tar.gz", basedir); + securec_check_ss_c(errorno, "", ""); + ztarfile = openGzFile(filename, compresslevel, "ab"); + } else +#endif + { + errorno = snprintf_s(filename, sizeof(filename), sizeof(filename) - 1, "%s/base.tar", basedir); + // basdir has been realpath before + securec_check_ss_c(errorno, "", ""); + tarfile = fopen(filename, "ab"); + } + } + + /* chkptName header */ + copybuf = (char*)xmalloc0(TAR_BLOCK_SIZE); + + int chkptDirLen = strlen(chkptName); + errorno = strcpy_s(copybuf + 2, chkptDirLen + 1, chkptName); + securec_check_ss_c(errorno, "", ""); + + copybuf[0] = '.'; + copybuf[1] = '/'; + copybuf[chkptDirLen + 2] = '/'; + copybuf[TAR_FILE_TYPE] = TAR_TYPE_DICTORY; + + for (int i = 0; i < 11; i++) { + copybuf[TAR_LEN_LEFT + i] = '0'; + } + errorno = sprintf_s(©buf[TAR_FILE_MODE], TAR_BLOCK_SIZE - TAR_FILE_MODE, "%07o ", FILE_PERMISSION); + securec_check_ss_c(errorno, "", ""); + +#ifdef HAVE_LIBZ + if (ztarfile != NULL) { + if (!writeGzFile(ztarfile, copybuf, TAR_BLOCK_SIZE)) { + fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); + disconnect_and_exit(1); + } + } else +#endif + { + if (fwrite(copybuf, TAR_BLOCK_SIZE, 1, tarfile) != 1) { + fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); + disconnect_and_exit(1); + } + } + + while (1) { + int r; + + if (copybuf != NULL) { + PQfreemem(copybuf); + copybuf = NULL; + } + + r = PQgetCopyData(conn, ©buf, 0); + if (r == -1) { +#ifdef HAVE_LIBZ + if (ztarfile != NULL) { + if (gzclose(ztarfile) != 0) { + fprintf(stderr, + _("%s: could not close compressed file \"%s\": %s\n"), + progname, + filename, + get_gz_error(ztarfile)); + disconnect_and_exit(1); + } + } else +#endif + { + if (strcmp(basedir, "-") != 0) { + if (fclose(tarfile) != 0) { + fprintf( + stderr, _("%s: could not close file \"%s\": %s\n"), progname, filename, strerror(errno)); + tarfile = NULL; + disconnect_and_exit(1); + } + tarfile = NULL; + } + } + break; + } else if (r == TAR_READ_ERROR) { + fprintf(stderr, "%s: could not read COPY data: %s", progname, PQerrorMessage(conn)); + disconnect_and_exit(1); + } + if (current_len_left == 0 && current_padding == 0) { + /* No current file, so this must be the header for a new file */ + if (r != TAR_BLOCK_SIZE) { + fprintf(stderr, _("%s: invalid tar block header size: %d\n"), progname, r); + disconnect_and_exit(1); + } + + /* new file */ + int filemode; + totaldone += TAR_BLOCK_SIZE; + if (sscanf_s(copybuf + TAR_LEN_LEFT, "%201o", ¤t_len_left) != 1) { + fprintf(stderr, "%s: could not parse file size\n", progname); + disconnect_and_exit(1); + } + + /* Set permissions on the file */ + if (sscanf_s(©buf[TAR_FILE_MODE], "%07o ", (unsigned int*)&filemode) != 1) { + fprintf(stderr, "%s: could not parse file mode\n", progname); + disconnect_and_exit(1); + } + + /* + * All files are padded up to 512 bytes + */ + current_padding = ((current_len_left + TAR_FILE_PADDING) & ~TAR_FILE_PADDING) - current_len_left; + /* + * First part of header is zero terminated filename. + * when getting a checkpoint, file name can be either + * the control file (written to base_dir), or a checkpoint + * file (written to base_dir/chkpt_) + */ + if (strstr(copybuf, "mot.ctrl")) { + errorno = + snprintf_s(filename, sizeof(filename), sizeof(filename) - 1, "%s/%s", current_path, "mot.ctrl"); + securec_check_ss_c(errorno, "", ""); + } else { + char* chkptOffset = strstr(copybuf, chkptName); + if (chkptOffset) { + errorno = snprintf_s( + filename, sizeof(filename), sizeof(filename) - 1, "%s/%s", current_path, chkptOffset); + securec_check_ss_c(errorno, "", ""); + } else { + errorno = + snprintf_s(filename, sizeof(filename), sizeof(filename) - 1, "%s/%s", current_path, copybuf); + securec_check_ss_c(errorno, "", ""); + } + } + + if (filename[strlen(filename) - 1] == '/') { + filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */ + strcpy_s(copybuf, strlen(filename), filename); + continue; /* directory or link handled */ + } +#ifdef HAVE_LIBZ + if (ztarfile != NULL) { + if (!writeGzFile(ztarfile, copybuf, r)) { + fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); + disconnect_and_exit(1); + } + } else +#endif + { + if (fwrite(copybuf, r, 1, tarfile) != 1) { + fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); + disconnect_and_exit(1); + } + } + if (current_len_left == 0) { + continue; + } + } else { + /* + * Continuing blocks in existing file + */ + if (current_len_left == 0 && r == current_padding) { +#ifdef HAVE_LIBZ + if (ztarfile != NULL) { + if (!writeGzFile(ztarfile, copybuf, current_padding)) { + fprintf( + stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); + disconnect_and_exit(1); + } + } else +#endif + { + if (fwrite(copybuf, current_padding, 1, tarfile) != 1) { + fprintf( + stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); + disconnect_and_exit(1); + } + } + totaldone += r; + current_padding -= r; + continue; + } + + totaldone += r; +#ifdef HAVE_LIBZ + if (ztarfile != NULL) { + if (!writeGzFile(ztarfile, copybuf, r)) { + fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); + disconnect_and_exit(1); + } + } else +#endif + { + if (fwrite(copybuf, r, 1, tarfile) != 1) { + fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); + disconnect_and_exit(1); + } + } + if (current_len_left == 0) { + continue; + } + current_len_left -= r; + if (current_len_left == 0 && current_padding == 0) { + /* + * Received the last block, and there is no padding to be + * expected. Close the file and move on to the next tar + * header. + */ + fclose(file); + file = NULL; + continue; + } + } /* continuing data in existing file */ + } /* loop over all data blocks */ + if (tarfile != NULL) { + fclose(tarfile); + tarfile = NULL; + } + + if (copybuf != NULL) { + PQfreemem(copybuf); + copybuf = NULL; + } + + return; +} + /* * Receive a tar format stream from the connection to the server, and unpack * the contents of it into a directory. Only files, directories and @@ -154,11 +440,8 @@ static void MotReceiveAndUnpackTarFile(const char* basedir, const char* chkptNam */ filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */ if (mkdir(filename, S_IRWXU) != 0) { - fprintf(stderr, - "%s: could not create directory \"%s\": %s\n", - progname, - filename, - strerror(errno)); + fprintf( + stderr, "%s: could not create directory \"%s\": %s\n", progname, filename, strerror(errno)); disconnect_and_exit(1); } #ifndef WIN32 @@ -272,7 +555,8 @@ static void MotReceiveAndUnpackTarFile(const char* basedir, const char* chkptNam * @param controls verbose output * @return Boolean value denoting success or failure. */ -void FetchMotCheckpoint(const char* basedir, PGconn* fetchConn, const char* progname, bool verbose) +void FetchMotCheckpoint( + const char* basedir, PGconn* fetchConn, const char* progname, bool verbose, const char format, int compresslevel) { PGresult* res = NULL; const char* fetchQuery = "FETCH_MOT_CHECKPOINT"; @@ -333,17 +617,24 @@ void FetchMotCheckpoint(const char* basedir, PGconn* fetchConn, const char* prog fprintf(stderr, "%s: mot checkpoint directory: %s\n", progname, dirName); } - if (stat(dirName, &fileStat) < 0) { - if (pg_mkdir_p(dirName, S_IRWXU) == -1) { - fprintf(stderr, "%s: could not create directory \"%s\": %s\n", progname, dirName, strerror(errno)); + if (format == 'p') { + if (stat(dirName, &fileStat) < 0) { + if (pg_mkdir_p(dirName, S_IRWXU) == -1) { + fprintf(stderr, "%s: could not create directory \"%s\": %s\n", progname, dirName, strerror(errno)); + exit(1); + } + } else { + fprintf(stderr, "%s: directory \"%s\" already exists, please remove it and try again\n", progname, + dirName); exit(1); } + MotReceiveAndUnpackTarFile(basedir, chkptName, fetchConn, progname); + } else if (format == 't') { + MotReceiveAndAppendTarFile(basedir, chkptName, fetchConn, progname, compresslevel); } else { - fprintf(stderr, "%s: directory \"%s\" already exists, please remove it and try again\n", progname, dirName); + fprintf(stderr, "%s: unsupport format type: \"%c\".\n", progname, format); exit(1); } - - MotReceiveAndUnpackTarFile(basedir, chkptName, fetchConn, progname); if (verbose) { fprintf(stderr, "%s: finished fetching mot checkpoint\n", progname); } diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 388a47024eaec18026a244c3b76e93e4ad001c74..d2a3f58825fc256b0fe4c60b69db8eb39907f1cc 100644 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -290,7 +290,8 @@ static char* get_string_by_sync_mode(bool syncmode); static void free_ctl(); extern int GetLengthAndCheckReplConn(const char* ConnInfoList); extern BuildErrorCode gs_increment_build(char* pgdata, const char* connstr, const uint32 term); -extern void FetchMotCheckpoint(const char* basedir, PGconn* fetchConn, const char* progname, bool verbose); +extern void FetchMotCheckpoint(const char* basedir, PGconn* fetchConn, const char* progname, bool verbose, + const char format = 'p', const int compresslevel = 0); extern char* GetOptionValueFromFile(const char* fileName, const char* option); void check_input_for_security(char* input_env_value) @@ -2268,7 +2269,7 @@ static void do_query(void) } static bool check_pid_exists(pgpid_t pid) -{ +{ FILE* fp = NULL; char buf[1024] = {0}; char command[128] = {0}; diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 8a9a9e0f644b41bb11f24e2cfa7b3db33e5ef1be..d97e44aecc524d385ab9978095eabb751f86b65d 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -94,7 +94,12 @@ typedef enum { } RecoveryTargetType; /* WAL levels */ -typedef enum WalLevel { WAL_LEVEL_MINIMAL = 0, WAL_LEVEL_ARCHIVE, WAL_LEVEL_HOT_STANDBY, WAL_LEVEL_LOGICAL } WalLevel; +typedef enum WalLevel { + WAL_LEVEL_MINIMAL = 0, + WAL_LEVEL_ARCHIVE, + WAL_LEVEL_HOT_STANDBY, + WAL_LEVEL_LOGICAL +} WalLevel; #define XLogArchivingActive() \ (u_sess->attr.attr_common.XLogArchiveMode && g_instance.attr.attr_storage.wal_level >= WAL_LEVEL_ARCHIVE) diff --git a/src/include/gs_tar_const.h b/src/include/gs_tar_const.h new file mode 100644 index 0000000000000000000000000000000000000000..ca610c5793671cbfdd82efdfd3bff15fa7f3ab85 --- /dev/null +++ b/src/include/gs_tar_const.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * + * + * IDENTIFICATION + * src/include/gs_tar_const.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef GS_TAR_CONST_H_ +#define GS_TAR_CONST_H_ + +#ifdef HAVE_LIBZ +#include "zlib.h" +#endif + +/* gs_tar offset */ +static const int TAR_LEN_LEFT = 1048; +static const int TAR_FILE_PADDING = 511; /* All files are padded up to 512 bytes */ +static const int TAR_FILE_TYPE = 1080; /* file type */ +static const int TAR_BLOCK_SIZE = 2560; /* gs_tar block size */ +static const int TAR_FILE_MODE = 1024; + +/* fileStream result */ +static const int TAR_READ_ERROR = -2; + +/* file type */ +static const char TAR_TYPE_DICTORY = '5'; +static const int FILE_PERMISSION = 16832; + +#ifdef HAVE_LIBZ +static const char* get_gz_error(gzFile gzf) +{ + int errnum; + const char* errmsg = NULL; + + errmsg = gzerror(gzf, &errnum); + if (errnum == Z_ERRNO) + return strerror(errno); + else + return errmsg; +} + +static gzFile openGzFile(const char* filename, int compresslevel, const char* mode = "wb") +{ + gzFile ztarfile = gzopen(filename, mode); + if (gzsetparams(ztarfile, compresslevel, Z_DEFAULT_STRATEGY) != Z_OK) { + return NULL; + } + return ztarfile; +} + +static bool writeGzFile(gzFile ztarfile, char* copybuf, int buf_size) +{ + if (gzwrite(ztarfile, copybuf, buf_size) != buf_size) { + return false; + } + return true; +} +#endif + +#endif /* GS_TAR_CONST_H_ */ \ No newline at end of file