diff --git a/Makefile b/Makefile index 2b663f59b..54fd6cb10 100644 --- a/Makefile +++ b/Makefile @@ -36,29 +36,28 @@ OBJS := src/utils/configuration.o src/utils/json.o src/utils/logger.o \ OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \ src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/stream.o \ - src/util.o src/validate.o src/datapagemap.o src/catchup.o + src/util.o src/validate.o src/datapagemap.o src/catchup.o \ + src/compatibility/pg-11.o # sources borrowed from postgresql (paths are relative to pg top dir) BORROWED_H_SRC := \ - src/include/portability/instr_time.h \ src/bin/pg_basebackup/receivelog.h \ src/bin/pg_basebackup/streamutil.h \ src/bin/pg_basebackup/walmethods.h BORROWED_C_SRC := \ src/backend/access/transam/xlogreader.c \ - src/backend/utils/hash/pg_crc.c \ src/bin/pg_basebackup/receivelog.c \ src/bin/pg_basebackup/streamutil.c \ src/bin/pg_basebackup/walmethods.c -OBJS += src/fu_util/impl/ft_impl.o src/fu_util/impl/fo_impl.o - BORROW_DIR := src/borrowed BORROWED_H := $(addprefix $(BORROW_DIR)/, $(notdir $(BORROWED_H_SRC))) BORROWED_C := $(addprefix $(BORROW_DIR)/, $(notdir $(BORROWED_C_SRC))) OBJS += $(patsubst %.c, %.o, $(BORROWED_C)) EXTRA_CLEAN := $(BORROWED_H) $(BORROWED_C) $(BORROW_DIR) borrowed.mk +OBJS += src/fu_util/impl/ft_impl.o src/fu_util/impl/fo_impl.o + # off-source build support ifneq ($(abspath $(CURDIR))/, $(top_pbk_srcdir)) VPATH := $(top_pbk_srcdir) @@ -87,7 +86,6 @@ override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS} # additional dependencies on borrowed files -src/archive.o: $(BORROW_DIR)/instr_time.h src/backup.o src/catchup.o src/pg_probackup.o: $(BORROW_DIR)/streamutil.h src/stream.o $(BORROW_DIR)/receivelog.o $(BORROW_DIR)/streamutil.o $(BORROW_DIR)/walmethods.o: $(BORROW_DIR)/receivelog.h $(BORROW_DIR)/receivelog.h: $(BORROW_DIR)/walmethods.h @@ -97,7 +95,7 @@ borrowed.mk: $(firstword $(MAKEFILE_LIST)) $(file >$@,# This file is autogenerated. Do not edit!) $(foreach borrowed_file, $(BORROWED_H_SRC) $(BORROWED_C_SRC), \ $(file >>$@,$(addprefix $(BORROW_DIR)/, $(notdir $(borrowed_file))): | $(CURDIR)/$(BORROW_DIR)/ $(realpath $(top_srcdir)/$(borrowed_file))) \ - $(file >>$@,$(shell echo "\t"'$$(LN_S) $(realpath $(top_srcdir)/$(borrowed_file)) $$@')) \ + $(file >>$@,$(shell echo " "'$$(LN_S) $(realpath $(top_srcdir)/$(borrowed_file)) $$@')) \ ) include borrowed.mk diff --git a/src/archive.c b/src/archive.c index 0ebe5e504..2cd62e20e 100644 --- a/src/archive.c +++ b/src/archive.c @@ -11,7 +11,7 @@ #include #include "pg_probackup.h" #include "utils/thread.h" -#include "instr_time.h" +#include "portability/instr_time.h" static int push_file_internal(const char *wal_file_name, const char *pg_xlog_dir, @@ -148,7 +148,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg n_threads = parray_num(batch_files); elog(INFO, "pg_probackup archive-push WAL file: %s, " - "threads: %i/%i, batch: %lu/%i, compression: %s", + "threads: %i/%i, batch: %zu/%i, compression: %s", wal_file_name, n_threads, num_threads, parray_num(batch_files), batch_size, is_compress ? "zlib" : "none"); @@ -279,7 +279,7 @@ push_files(void *arg) int rc; archive_push_arg *args = (archive_push_arg *) arg; - my_thread_num = args->thread_num; + set_my_thread_num(args->thread_num); for (i = 0; i < parray_num(args->files); i++) { @@ -553,17 +553,17 @@ push_file_internal(const char *wal_file_name, const char *pg_xlog_dir, /* enable streaming compression */ if (is_compress) -#ifdef HAVE_LIBZ { +#ifdef HAVE_LIBZ pioFilter_i flt = pioGZCompressFilter(compress_level); err = pioCopy($reduce(pioWriteFlush, out), $reduce(pioRead, in), flt); - } - else #else - elog(ERROR, "Compression is requested, but not compiled it"); + elog(ERROR, "Compression is requested, but not compiled it"); #endif + } + else { err = pioCopy($reduce(pioWriteFlush, out), $reduce(pioRead, in)); @@ -1011,7 +1011,7 @@ get_files(void *arg) char from_fullpath[MAXPGPATH]; archive_get_arg *args = (archive_get_arg *) arg; - my_thread_num = args->thread_num; + set_my_thread_num(args->thread_num); for (i = 0; i < parray_num(args->files); i++) { diff --git a/src/backup.c b/src/backup.c index 15f1a4d1c..9401d6d72 100644 --- a/src/backup.c +++ b/src/backup.c @@ -17,7 +17,6 @@ #include "pgtar.h" #include "streamutil.h" -#include #include #include @@ -1725,7 +1724,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c if (S_ISREG(file->mode)) { - file->crc = pgFileGetCRC(full_filename, true, false); + file->crc = pgFileGetCRC32C(full_filename, false); file->write_size = file->size; file->uncompressed_size = file->size; @@ -2179,6 +2178,7 @@ check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) PGresult *res; int i = 0; int j = 0; + int ntups; char *tablespace_path = NULL; char *query = "SELECT pg_catalog.pg_tablespace_location(oid) " "FROM pg_catalog.pg_tablespace " @@ -2190,7 +2190,8 @@ check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) if (!res) elog(ERROR, "Failed to get list of tablespaces"); - for (i = 0; i < res->ntups; i++) + ntups = PQntuples(res); + for (i = 0; i < ntups; i++) { tablespace_path = PQgetvalue(res, i, 0); Assert (strlen(tablespace_path) > 0); diff --git a/src/catalog.c b/src/catalog.c index b4be159d1..13a522195 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -13,7 +13,6 @@ #include #include -#include #include #include "utils/file.h" @@ -423,8 +422,8 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) /* complain every fifth interval */ if ((ntries % LOG_FREQ) == 0) { - elog(WARNING, "Process %d is using backup %s, and is still running", - encoded_pid, backup_id); + elog(WARNING, "Process %lld is using backup %s, and is still running", + (long long)encoded_pid, backup_id); elog(WARNING, "Waiting %u seconds on exclusive lock for backup %s", ntries, backup_id); @@ -438,8 +437,8 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) else { if (errno == ESRCH) - elog(WARNING, "Process %d which used backup %s no longer exists", - encoded_pid, backup_id); + elog(WARNING, "Process %lld which used backup %s no longer exists", + (long long)encoded_pid, backup_id); else elog(ERROR, "Failed to send signal 0 to a process %d: %s", encoded_pid, strerror(errno)); @@ -468,7 +467,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) /* * Successfully created the file, now fill it. */ - snprintf(buffer, sizeof(buffer), "%d\n", my_pid); + snprintf(buffer, sizeof(buffer), "%lld\n", (long long)my_pid); errno = 0; if (fio_write(fd, buffer, strlen(buffer)) != strlen(buffer)) @@ -575,8 +574,8 @@ wait_shared_owners(pgBackup *backup) /* complain from time to time */ if ((ntries % LOG_FREQ) == 0) { - elog(WARNING, "Process %d is using backup %s in shared mode, and is still running", - encoded_pid, base36enc(backup->start_time)); + elog(WARNING, "Process %lld is using backup %s in shared mode, and is still running", + (long long)encoded_pid, base36enc(backup->start_time)); elog(WARNING, "Waiting %u seconds on lock for backup %s", ntries, base36enc(backup->start_time)); @@ -588,8 +587,8 @@ wait_shared_owners(pgBackup *backup) continue; } else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); + elog(ERROR, "Failed to send signal 0 to a process %lld: %s", + (long long)encoded_pid, strerror(errno)); /* locker is dead */ break; @@ -606,8 +605,8 @@ wait_shared_owners(pgBackup *backup) /* some shared owners are still alive */ if (ntries <= 0) { - elog(WARNING, "Cannot to lock backup %s in exclusive mode, because process %u owns shared lock", - base36enc(backup->start_time), encoded_pid); + elog(WARNING, "Cannot to lock backup %s in exclusive mode, because process %llu owns shared lock", + base36enc(backup->start_time), (long long)encoded_pid); return 1; } @@ -662,11 +661,11 @@ grab_shared_lock_file(pgBackup *backup) * Somebody is still using this backup in shared mode, * copy this pid into a new file. */ - buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); + buffer_len += snprintf(buffer+buffer_len, 4096, "%llu\n", (long long)encoded_pid); } else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); + elog(ERROR, "Failed to send signal 0 to a process %lld: %s", + (long long)encoded_pid, strerror(errno)); } if (fp_in) @@ -686,7 +685,7 @@ grab_shared_lock_file(pgBackup *backup) } /* add my own pid */ - buffer_len += snprintf(buffer+buffer_len, sizeof(buffer), "%u\n", my_pid); + buffer_len += snprintf(buffer+buffer_len, sizeof(buffer), "%llu\n", (long long)my_pid); /* write out the collected PIDs to temp lock file */ fwrite(buffer, 1, buffer_len, fp_out); @@ -784,11 +783,11 @@ release_shared_lock_file(const char *backup_dir) * Somebody is still using this backup in shared mode, * copy this pid into a new file. */ - buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); + buffer_len += snprintf(buffer+buffer_len, 4096, "%llu\n", (long long)encoded_pid); } else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); + elog(ERROR, "Failed to send signal 0 to a process %lld: %s", + (long long)encoded_pid, strerror(errno)); } if (ferror(fp_in)) @@ -1067,7 +1066,7 @@ get_backup_filelist(pgBackup *backup, bool strict) files = parray_new(); - INIT_FILE_CRC32(true, content_crc); + INIT_CRC32C(content_crc); while (fgets(buf, lengthof(buf), fp)) { @@ -1089,7 +1088,7 @@ get_backup_filelist(pgBackup *backup, bool strict) hdr_size; pgFile *file; - COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); + COMP_CRC32C(content_crc, buf, strlen(buf)); get_control_value_str(buf, "path", path, sizeof(path),true); get_control_value_int64(buf, "size", &write_size, true); @@ -1141,7 +1140,7 @@ get_backup_filelist(pgBackup *backup, bool strict) parray_append(files, file); } - FIN_FILE_CRC32(true, content_crc); + FIN_CRC32C(content_crc); if (ferror(fp)) elog(ERROR, "Failed to read from file: \"%s\"", backup_filelist_path); @@ -2538,7 +2537,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, setvbuf(out, buf, _IOFBF, BUFFERSZ); if (sync) - INIT_FILE_CRC32(true, backup->content_crc); + INIT_CRC32C(backup->content_crc); /* print each file in the list */ for (i = 0; i < parray_num(files); i++) @@ -2606,13 +2605,13 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, sprintf(line+len, "}\n"); if (sync) - COMP_FILE_CRC32(true, backup->content_crc, line, strlen(line)); + COMP_CRC32C(backup->content_crc, line, strlen(line)); fprintf(out, "%s", line); } if (sync) - FIN_FILE_CRC32(true, backup->content_crc); + FIN_CRC32C(backup->content_crc); if (fflush(out) != 0) elog(ERROR, "Cannot flush file list \"%s\": %s", diff --git a/src/catchup.c b/src/catchup.c index 0f6e36b13..08bc039f9 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -17,7 +17,6 @@ #include "pgtar.h" #include "streamutil.h" -#include #include #include @@ -138,8 +137,8 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, } else if (pid > 1) /* postmaster is up */ { - elog(ERROR, "Postmaster with pid %u is running in destination directory \"%s\"", - pid, dest_pgdata); + elog(ERROR, "Postmaster with pid %lld is running in destination directory \"%s\"", + (long long)pid, dest_pgdata); } } @@ -161,15 +160,15 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, source_id = get_system_identifier(FIO_DB_HOST, source_pgdata, false); /* same as instance_config.system_identifier */ if (source_conn_id != source_id) - elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", - source_conn_id, source_pgdata, source_id); + elog(ERROR, "Database identifiers mismatch: we connected to DB id %llu, but in \"%s\" we found id %llu", + (long long)source_conn_id, source_pgdata, (long long)source_id); if (current.backup_mode != BACKUP_MODE_FULL) { dest_id = get_system_identifier(FIO_LOCAL_HOST, dest_pgdata, false); if (source_conn_id != dest_id) - elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", - source_conn_id, dest_pgdata, dest_id); + elog(ERROR, "Database identifiers mismatch: we connected to DB id %llu, but in \"%s\" we found id %llu", + (long long)source_conn_id, dest_pgdata, (long long)dest_id); } } @@ -230,6 +229,7 @@ catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn) { PGresult *res; int i; + int ntups; char *tablespace_path = NULL; const char *linked_path = NULL; char *query = "SELECT pg_catalog.pg_tablespace_location(oid) " @@ -241,7 +241,8 @@ catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn) if (!res) elog(ERROR, "Failed to get list of tablespaces"); - for (i = 0; i < res->ntups; i++) + ntups = PQntuples(res); + for (i = 0; i < ntups; i++) { tablespace_path = PQgetvalue(res, i, 0); Assert (strlen(tablespace_path) > 0); @@ -438,7 +439,7 @@ catchup_thread_runner(void *arg) if (file->write_size == BYTES_INVALID) { - elog(LOG, "Skipping the unchanged file: \"%s\", read %li bytes", from_fullpath, file->read_size); + elog(LOG, "Skipping the unchanged file: \"%s\", read %zu bytes", from_fullpath, file->read_size); continue; } diff --git a/src/checkdb.c b/src/checkdb.c index 177fc3cc7..f1a5fcf78 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -16,7 +16,6 @@ #include "pg_probackup.h" -#include #include #include @@ -293,7 +292,8 @@ check_indexes(void *arg) int i; check_indexes_arg *arguments = (check_indexes_arg *) arg; int n_indexes = 0; - my_thread_num = arguments->thread_num; + + set_my_thread_num(arguments->thread_num); if (arguments->index_list) n_indexes = parray_num(arguments->index_list); diff --git a/src/compatibility/pg-11.c b/src/compatibility/pg-11.c new file mode 100644 index 000000000..52f4b551c --- /dev/null +++ b/src/compatibility/pg-11.c @@ -0,0 +1,98 @@ +/*------------------------------------------------------------------------- + * + * pg-11.c + * PostgreSQL <= 11 compatibility + * + * Portions Copyright (c) 2022, Postgres Professional + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + *------------------------------------------------------------------------- + */ + +#include + +#if PG_VERSION_NUM < 120000 + +#include "c.h" +#include "utils/pg_crc.h" + +/* From postgresql src/backend/utils/hash/pg_crc.c */ + +/* + * Lookup table for calculating CRC-32 using Sarwate's algorithm. + * + * This table is based on the polynomial + * x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1. + * (This is the same polynomial used in Ethernet checksums, for instance.) + * Using Williams' terms, this is the "normal", not "reflected" version. + */ + +const uint32 pg_crc32_table[256] = { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, + 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, + 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, + 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, + 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, + 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, + 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, + 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, + 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, + 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, + 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, + 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, + 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, + 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, + 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, + 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, + 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, + 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, + 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, + 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, + 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, + 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D +}; + +#endif diff --git a/src/compatibility/pg-11.h b/src/compatibility/pg-11.h new file mode 100644 index 000000000..63a83070a --- /dev/null +++ b/src/compatibility/pg-11.h @@ -0,0 +1,67 @@ +/*------------------------------------------------------------------------- + * + * pg-11.h + * PostgreSQL <= 11 compatibility + * + * Copyright (c) 2022, Postgres Professional + * + * When PG-11 reaches the end of support, we will need to remove + * *_CRC32_COMPAT macros and use *_CRC32C instead. + * And this file will be removed. + *------------------------------------------------------------------------- + */ + +#ifndef PG11_COMPAT_H +#define PG11_COMPAT_H + +#include "utils/pgut.h" + +#if PG_VERSION_NUM >= 120000 + +#define INIT_CRC32_COMPAT(backup_version, crc) \ +do { \ + Assert(backup_version >= 20025); \ + INIT_CRC32C(crc); \ +} while (0) + +#define COMP_CRC32_COMPAT(backup_version, crc, data, len) \ +do { \ + Assert(backup_version >= 20025); \ + COMP_CRC32C((crc), (data), (len)); \ +} while (0) + +#define FIN_CRC32_COMPAT(backup_version, crc) \ +do { \ + Assert(backup_version >= 20025); \ + FIN_CRC32C(crc); \ +} while (0) + +#else /* PG_VERSION_NUM < 120000 */ + +#define INIT_CRC32_COMPAT(backup_version, crc) \ +do { \ + if (backup_version <= 20021 || backup_version >= 20025) \ + INIT_CRC32C(crc); \ + else \ + INIT_TRADITIONAL_CRC32(crc); \ +} while (0) + +#define COMP_CRC32_COMPAT(backup_version, crc, data, len) \ +do { \ + if (backup_version <= 20021 || backup_version >= 20025) \ + COMP_CRC32C((crc), (data), (len)); \ + else \ + COMP_TRADITIONAL_CRC32(crc, data, len); \ +} while (0) + +#define FIN_CRC32_COMPAT(backup_version, crc) \ +do { \ + if (backup_version <= 20021 || backup_version >= 20025) \ + FIN_CRC32C(crc); \ + else \ + FIN_TRADITIONAL_CRC32(crc); \ +} while (0) + +#endif /* PG_VERSION_NUM < 120000 */ + +#endif /* PG11_COMPAT_H */ diff --git a/src/data.c b/src/data.c index 17ae4b91a..9439e9d48 100644 --- a/src/data.c +++ b/src/data.c @@ -16,7 +16,6 @@ #include "utils/file.h" #include -#include #ifdef HAVE_LIBZ #include @@ -24,6 +23,9 @@ #include "utils/thread.h" +/* for crc32_compat macros */ +#include "compatibility/pg-11.h" + /* Union to ease operations on relation pages */ typedef struct DataPage { @@ -32,7 +34,7 @@ typedef struct DataPage } DataPage; static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader *bph, - pg_crc32 *crc, bool use_crc32c); + pg_crc32 *crc, uint32 backup_version); #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ @@ -205,12 +207,12 @@ get_header_errormsg(Page page, char **errormsg) if (PageGetPageSize(phdr) != BLCKSZ) snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "page size %lu is not equal to block size %u", + "page size %zu is not equal to block size %u", PageGetPageSize(phdr), BLCKSZ); else if (phdr->pd_lower < SizeOfPageHeaderData) snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "pd_lower %i is less than page header size %lu", + "pd_lower %i is less than page header size %zu", phdr->pd_lower, SizeOfPageHeaderData); else if (phdr->pd_lower > phdr->pd_upper) @@ -230,7 +232,7 @@ get_header_errormsg(Page page, char **errormsg) else if (phdr->pd_special != MAXALIGN(phdr->pd_special)) snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "pd_special %i is misaligned, expected %lu", + "pd_special %i is misaligned, expected %zu", phdr->pd_special, MAXALIGN(phdr->pd_special)); else if (phdr->pd_flags & ~PD_VALID_FLAG_BITS) @@ -448,7 +450,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, write_buffer_size = compressed_size + sizeof(BackupPageHeader); /* Update CRC */ - COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size); + COMP_CRC32C(*crc, write_buffer, write_buffer_size); /* write data page */ if (fio_fwrite(out, write_buffer, write_buffer_size) != write_buffer_size) @@ -529,7 +531,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat file->read_size = 0; file->write_size = 0; file->uncompressed_size = 0; - INIT_FILE_CRC32(true, file->crc); + INIT_CRC32C(file->crc); /* * Read each page, verify checksum and write it to backup. @@ -628,7 +630,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat cleanup: /* finish CRC calculation */ - FIN_FILE_CRC32(true, file->crc); + FIN_CRC32C(file->crc); /* dump page headers */ write_page_headers(headers, file, hdr_map, is_merge); @@ -805,7 +807,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, file->crc = fio_get_crc32(FIO_DB_HOST, from_fullpath, false); /* ...and checksum is the same... */ - if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) + if (EQ_CRC32C(file->crc, prev_file->crc)) { file->write_size = BYTES_INVALID; return; /* ...skip copying file. */ @@ -1018,7 +1020,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * or when merging something. Align read_len only when restoring * or merging old backups. */ - if (get_page_header(in, from_fullpath, &(page).bph, NULL, false)) + if (get_page_header(in, from_fullpath, &(page).bph, NULL, backup_version)) { cur_pos_in += sizeof(BackupPageHeader); @@ -1197,7 +1199,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers datapagemap_add(map, blknum); } - elog(LOG, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); + elog(LOG, "Copied file \"%s\": %zu bytes", from_fullpath, write_len); return write_len; } @@ -1241,7 +1243,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, pg_free(buf); - elog(LOG, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); + elog(LOG, "Copied file \"%s\": %llu bytes", from_fullpath, (long long)file->write_size); } size_t @@ -1318,9 +1320,9 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath); if (tmp_file->write_size <= 0) - elog(ERROR, "Full copy of non-data file has invalid size: %li. " + elog(ERROR, "Full copy of non-data file has invalid size: %lli. " "Metadata corruption in backup %s in file: \"%s\"", - tmp_file->write_size, base36enc(tmp_backup->start_time), + (long long)tmp_file->write_size, base36enc(tmp_backup->start_time), to_fullpath); /* incremental restore */ @@ -1389,7 +1391,7 @@ backup_non_data_file_internal(const char *from_fullpath, ssize_t read_len = 0; char *buf = NULL; - INIT_FILE_CRC32(true, file->crc); + INIT_CRC32C(file->crc); /* reset size summary */ file->read_size = 0; @@ -1485,7 +1487,7 @@ backup_non_data_file_internal(const char *from_fullpath, strerror(errno)); /* update CRC */ - COMP_FILE_CRC32(true, file->crc, buf, read_len); + COMP_CRC32C(file->crc, buf, read_len); file->read_size += read_len; } @@ -1501,7 +1503,7 @@ backup_non_data_file_internal(const char *from_fullpath, cleanup: /* finish CRC calculation and store into pgFile */ - FIN_FILE_CRC32(true, file->crc); + FIN_CRC32C(file->crc); if (in && fclose(in)) elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno)); @@ -1678,7 +1680,6 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, bool is_valid = true; FILE *in; pg_crc32 crc; - bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; BackupPageHeader2 *headers = NULL; int n_hdr = -1; off_t cur_pos_in = 0; @@ -1702,7 +1703,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, } /* calc CRC of backup file */ - INIT_FILE_CRC32(use_crc32c, crc); + INIT_CRC32_COMPAT(backup_version, crc); /* read and validate pages one by one */ while (true) @@ -1718,7 +1719,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during data file validation"); - /* newer backups have page headers in separate storage */ + /* newer backups (post 2.4.0) have page headers in separate storage */ if (headers) { n_hdr++; @@ -1747,10 +1748,10 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, cur_pos_in = headers[n_hdr].pos; } } - /* old backups rely on header located directly in data file */ + /* old backups (pre 2.4.0) rely on header located directly in data file */ else { - if (get_page_header(in, fullpath, &(compressed_page).bph, &crc, use_crc32c)) + if (get_page_header(in, fullpath, &(compressed_page).bph, &crc, backup_version)) { /* Backward compatibility kludge, TODO: remove in 3.0 * for some reason we padded compressed pages in old versions @@ -1790,9 +1791,9 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, cur_pos_in += read_len; if (headers) - COMP_FILE_CRC32(use_crc32c, crc, &compressed_page, read_len); + COMP_CRC32_COMPAT(backup_version, crc, &compressed_page, read_len); else - COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); + COMP_CRC32_COMPAT(backup_version, crc, compressed_page.data, read_len); if (compressed_size != BLCKSZ || page_may_be_compressed(compressed_page.data, file->compress_alg, @@ -1861,7 +1862,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, } } - FIN_FILE_CRC32(use_crc32c, crc); + FIN_CRC32_COMPAT(backup_version, crc); fclose(in); if (crc != file->crc) @@ -2017,7 +2018,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, /* Every page in data file contains BackupPageHeader, extract it */ bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, - pg_crc32 *crc, bool use_crc32c) + pg_crc32 *crc, uint32 backup_version) { /* read BackupPageHeader */ size_t read_len = fread(bph, 1, sizeof(BackupPageHeader), in); @@ -2032,11 +2033,11 @@ get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, return false; /* EOF found */ else if (read_len != 0 && feof(in)) elog(ERROR, - "Odd size page found at offset %ld of \"%s\"", - ftello(in), fullpath); + "Odd size page found at offset %lld of \"%s\"", + (long long)ftello(in), fullpath); else - elog(ERROR, "Cannot read header at offset %ld of \"%s\": %s", - ftello(in), fullpath, strerror(errno)); + elog(ERROR, "Cannot read header at offset %lld of \"%s\": %s", + (long long)ftello(in), fullpath, strerror(errno)); } /* In older versions < 2.4.0, when crc for file was calculated, header was @@ -2044,7 +2045,7 @@ get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, * the problem of backward compatibility for backups of old versions */ if (crc) - COMP_FILE_CRC32(use_crc32c, *crc, bph, read_len); + COMP_CRC32_COMPAT(backup_version, *crc, bph, read_len); if (bph->block == 0 && bph->compressed_size == 0) elog(ERROR, "Empty block in file \"%s\"", fullpath); @@ -2336,8 +2337,8 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, to_fullpath, strerror(errno)); if (ftruncate(fileno(out), file->size) == -1) - elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", - to_fullpath, file->size, strerror(errno)); + elog(ERROR, "Cannot ftruncate file \"%s\" to size %llu: %s", + to_fullpath, (long long)file->size, strerror(errno)); } } @@ -2363,6 +2364,8 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, * array of headers. * TODO: some access optimizations would be great here: * less fseeks, buffering, descriptor sharing, etc. + * + * Used for post 2.4.0 backups */ BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict) @@ -2437,14 +2440,14 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b } /* validate checksum */ - INIT_FILE_CRC32(true, hdr_crc); - COMP_FILE_CRC32(true, hdr_crc, headers, read_len); - FIN_FILE_CRC32(true, hdr_crc); + INIT_CRC32C(hdr_crc); + COMP_CRC32C(hdr_crc, headers, read_len); + FIN_CRC32C(hdr_crc); if (hdr_crc != file->hdr_crc) { elog(strict ? ERROR : WARNING, "Header map for file \"%s\" crc mismatch \"%s\" " - "offset: %llu, len: %lu, current: %u, expected: %u", + "offset: %llu, len: %zu, current: %u, expected: %u", file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); goto cleanup; } @@ -2486,9 +2489,9 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, read_len = (file->n_headers + 1) * sizeof(BackupPageHeader2); /* calculate checksums */ - INIT_FILE_CRC32(true, file->hdr_crc); - COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len); - FIN_FILE_CRC32(true, file->hdr_crc); + INIT_CRC32C(file->hdr_crc); + COMP_CRC32C(file->hdr_crc, headers, read_len); + FIN_CRC32C(file->hdr_crc); zheaders = pgut_malloc(read_len * 2); memset(zheaders, 0, read_len * 2); diff --git a/src/dir.c b/src/dir.c index 0bcd60169..165cc637f 100644 --- a/src/dir.c +++ b/src/dir.c @@ -19,7 +19,6 @@ #include "catalog/pg_tablespace.h" #include -#include #include #include "utils/configuration.h" @@ -203,26 +202,23 @@ pgFileInit(const char *rel_path) * obvious about it. */ pg_crc32 -pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) +pgFileGetCRC32C(const char *file_path, bool missing_ok) { FILE *fp; pg_crc32 crc = 0; char *buf; size_t len = 0; - INIT_FILE_CRC32(use_crc32c, crc); + INIT_CRC32C(crc); /* open file in binary read mode */ fp = fopen(file_path, PG_BINARY_R); if (fp == NULL) { - if (errno == ENOENT) + if (missing_ok && errno == ENOENT) { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } + FIN_CRC32C(crc); + return crc; } elog(ERROR, "Cannot open file \"%s\": %s", @@ -234,7 +230,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) buf = pgut_malloc(STDIO_BUFSIZE); /* calc CRC of file */ - for (;;) + do { if (interrupted) elog(ERROR, "interrupted during CRC calculation"); @@ -244,19 +240,75 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) if (ferror(fp)) elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); + COMP_CRC32C(crc, buf, len); + } + while (!feof(fp)); + + FIN_CRC32C(crc); + fclose(fp); + pg_free(buf); + + return crc; +} + +#if PG_VERSION_NUM < 120000 +/* + * Read the local file to compute its CRC using traditional algorithm. + * (*_TRADITIONAL_CRC32 macros) + * This was used only in version 2.0.22--2.0.24 + * And never used for PG >= 12 + * To be removed with end of PG-11 support + */ +pg_crc32 +pgFileGetCRC32(const char *file_path, bool missing_ok) +{ + FILE *fp; + pg_crc32 crc = 0; + char *buf; + size_t len = 0; + + INIT_TRADITIONAL_CRC32(crc); + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + if (missing_ok && errno == ENOENT) + { + FIN_TRADITIONAL_CRC32(crc); + return crc; + } - if (feof(fp)) - break; + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); } - FIN_FILE_CRC32(use_crc32c, crc); + /* disable stdio buffering */ + setvbuf(fp, NULL, _IONBF, BUFSIZ); + buf = pgut_malloc(STDIO_BUFSIZE); + + /* calc CRC of file */ + do + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + len = fread(buf, 1, STDIO_BUFSIZE, fp); + + if (ferror(fp)) + elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); + + COMP_TRADITIONAL_CRC32(crc, buf, len); + } + while (!feof(fp)); + + FIN_TRADITIONAL_CRC32(crc); fclose(fp); pg_free(buf); return crc; } +#endif /* PG_VERSION_NUM < 120000 */ /* * Read the local file to compute its CRC. @@ -265,7 +317,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) * obvious about it. */ pg_crc32 -pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) +pgFileGetCRC32Cgz(const char *file_path, bool missing_ok) { gzFile fp; pg_crc32 crc = 0; @@ -273,19 +325,16 @@ pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) int err; char *buf; - INIT_FILE_CRC32(use_crc32c, crc); + INIT_CRC32C(crc); /* open file in binary read mode */ fp = gzopen(file_path, PG_BINARY_R); if (fp == NULL) { - if (errno == ENOENT) + if (missing_ok && errno == ENOENT) { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } + FIN_CRC32C(crc); + return crc; } elog(ERROR, "Cannot open file \"%s\": %s", @@ -311,16 +360,16 @@ pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) { const char *err_str = NULL; - err_str = gzerror(fp, &err); - elog(ERROR, "Cannot read from compressed file %s", err_str); + err_str = gzerror(fp, &err); + elog(ERROR, "Cannot read from compressed file %s", err_str); } } /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); + COMP_CRC32C(crc, buf, len); } - FIN_FILE_CRC32(use_crc32c, crc); + FIN_CRC32C(crc); gzclose(fp); pg_free(buf); @@ -1758,7 +1807,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ /* Add metadata to backup_content.control */ file = pgFileNew(database_map_path, DATABASE_MAP, true, 0, FIO_BACKUP_HOST); - file->crc = pgFileGetCRC(database_map_path, true, false); + file->crc = pgFileGetCRC32C(database_map_path, false); file->write_size = file->size; file->uncompressed_size = file->read_size; diff --git a/src/fetch.c b/src/fetch.c index bbea7bffe..980bf531b 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -10,7 +10,6 @@ #include "pg_probackup.h" -#include #include /* diff --git a/src/fu_util/CMakeLists.txt b/src/fu_util/CMakeLists.txt index 6752d5dd2..a5426df42 100644 --- a/src/fu_util/CMakeLists.txt +++ b/src/fu_util/CMakeLists.txt @@ -5,6 +5,7 @@ set(CMAKE_C_STANDARD 99) set(CMAKE_C_EXTENSIONS true) include(CheckCSourceCompiles) +include(CheckFunctionExists) add_library(fu_utils impl/ft_impl.c impl/fo_impl.c) @@ -12,12 +13,21 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) target_link_libraries(fu_utils PRIVATE Threads::Threads) +if(CMAKE_USE_PTHREADS_INIT) + target_compile_definitions(fu_utils PRIVATE USE_PTHREADS) +else() + message(FATAL_ERROR "Need pthread support to build") +endif() + +CHECK_FUNCTION_EXISTS(strerror_r HAVE_STRERROR_R) + # Detect for installed beautiful https://github.com/ianlancetaylor/libbacktrace include_directories(.) if(NOT CMAKE_C_COMPILER MATCHES tcc) find_library(LIBBACKTRACE backtrace) if(LIBBACKTRACE) set(CMAKE_REQUIRED_LIBRARIES backtrace) + target_link_libraries(fu_utils PRIVATE backtrace) check_c_source_compiles(" #include int main(void) { @@ -30,11 +40,21 @@ if(NOT CMAKE_C_COMPILER MATCHES tcc) endif() endif() endif() +check_include_file(execinfo.h HAVE_EXECINFO_H) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fexceptions") + +if(HAVE_EXECINFO_H) + target_compile_definitions(fu_utils PRIVATE HAVE_EXECINFO_H) +endif() +if(HAVE_STRERROR_R) + target_compile_definitions(fu_utils PRIVATE HAVE_STRERROR_R) +endif() + configure_file(fu_utils_cfg.h.in fu_utils_cfg.h) target_include_directories(fu_utils INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}") target_include_directories(fu_utils PRIVATE "${PROJECT_BINARY_DIR}") -target_link_libraries(fu_utils PUBLIC backtrace) install(TARGETS fu_utils DESTINATION lib) install(FILES fm_util.h ft_util.h fo_obj.h @@ -43,4 +63,4 @@ install(FILES fm_util.h ft_util.h fo_obj.h DESTINATION include/fu_utils) install(FILES impl/ft_impl.h impl/fo_impl.h DESTINATION include/fu_utils/impl) -add_subdirectory(test) \ No newline at end of file +add_subdirectory(test) diff --git a/src/fu_util/fm_util.h b/src/fu_util/fm_util.h index 11d96682d..18a971aa7 100644 --- a/src/fu_util/fm_util.h +++ b/src/fu_util/fm_util.h @@ -24,6 +24,9 @@ /****************************************/ // LOGIC +#define fm_true 1 +#define fm_false 0 + #define fm_compl(v) fm_cat(fm_compl_, v) #define fm_compl_0 1 #define fm_compl_1 0 @@ -81,38 +84,56 @@ #define fm_tail(...) fm__tail(__VA_ARGS__) #define fm__tail(x, ...) __VA_ARGS__ -#define fm_or_default(...) \ - fm_iif(fm_va_01(__VA_ARGS__))(__VA_ARGS__) #define fm_va_single(...) fm__va_single(__VA_ARGS__, fm__comma) #define fm_va_many(...) fm__va_many(__VA_ARGS__, fm__comma) #define fm__va_single(x, y, ...) fm__va_result(y, 1, 0) #define fm__va_many(x, y, ...) fm__va_result(y, 0, 1) -#define fm__va_result(x, y, res, ...) res +#define fm__va_result(...) fm__va_result_fin(__VA_ARGS__) +#define fm__va_result_fin(x, y, res, ...) res #define fm_no_va fm_is_empty #define fm_va_01 fm_isnt_empty -#define fm_va_01n(...) fm_cat3(fm__va_01n_, fm__isnt_empty(__VA_ARGS__), fm_va_many(__VA_ARGS__)) -#define fm__va_01n_00 0 -#define fm__va_01n_10 1 -#define fm__va_01n_11 n -#if !__STRICT_ANSI__ +#ifndef FM_USE_STRICT + #if defined(__STRICT_ANSI__) || defined(_MSC_VER) /* well, clang-cl doesn't allow to distinguish std mode */ + #define FM_USE_STRICT + #endif +#endif + +#ifndef FM_USE_STRICT #define fm_is_empty(...) fm__is_empty(__VA_ARGS__) #define fm__is_empty(...) fm_va_single(~, ##__VA_ARGS__) #define fm_isnt_empty(...) fm__isnt_empty(__VA_ARGS__) #define fm__isnt_empty(...) fm_va_many(~, ##__VA_ARGS__) + +#define fm_va_01n(...) fm_cat3(fm__va_01n_, fm__isnt_empty(__VA_ARGS__), fm_va_many(__VA_ARGS__)) +#define fm__va_01n_00 0 +#define fm__va_01n_10 1 +#define fm__va_01n_11 n + +#define fm_when_isnt_empty(...) fm_cat(fm__when_, fm__isnt_empty(__VA_ARGS__)) #else #define fm_is_empty(...) fm_and(fm__is_emptyfirst(__VA_ARGS__), fm_va_single(__VA_ARGS__)) #define fm_isnt_empty(...) fm_nand(fm__is_emptyfirst(__VA_ARGS__), fm_va_single(__VA_ARGS__)) #define fm__is_emptyfirst(x, ...) fm_iif(fm_is_tuple(x))(0)(fm__is_emptyfirst_impl(x)) -#define fm__is_emptyfirst_impl(x,...) fm_tuple_2((\ - fm__is_emptyfirst_do1 x (fm__is_emptyfirst_do2), 1, 0)) +#define fm__is_emptyfirst_impl(x,...) fm__va_result(\ + fm__is_emptyfirst_do1 x (fm__is_emptyfirst_do2), 1, 0) #define fm__is_emptyfirst_do1(F) F() #define fm__is_emptyfirst_do2(...) , + +#define fm_when_isnt_empty(...) fm_cat(fm__when_, fm_isnt_empty(__VA_ARGS__)) + +#define fm_va_01n(...) fm_cat3(fm__va_01n_, fm__is_emptyfirst(__VA_ARGS__), fm_va_many(__VA_ARGS__)) +#define fm__va_01n_10 0 +#define fm__va_01n_00 1 +#define fm__va_01n_01 n +#define fm__va_01n_11 n #endif -#define fm_when_isnt_empty(...) fm_cat(fm__when_, fm__isnt_empty(__VA_ARGS__)) +#define fm_or_default(...) \ + fm_iif(fm_va_01(__VA_ARGS__))(__VA_ARGS__) + #define fm_va_comma(...) \ fm_when_isnt_empty(__VA_ARGS__)(fm__comma) #define fm_va_comma_fun(...) \ @@ -127,23 +148,6 @@ #define fm__is_tuple_help(...) , #define fm__is_tuple_(...) fm__is_tuple_choose(__VA_ARGS__) -#define fm_tuple_expand(x) fm_expand x -#define fm_tuple_tag(x) fm_head x -#define fm_tuple_data(x) fm_tail x -#define fm_tuple_0(x) fm_head x -#define fm_tuple_1(x) fm__tuple_1 x -#define fm__tuple_1(_0, _1, ...) _1 -#define fm_tuple_2(x) fm__tuple_2 x -#define fm__tuple_2(_0, _1, _2, ...) _2 - -#define fm_tuple_tag_or_0(x) fm__tuple_tag_or_0_(fm__tuple_tag_or_0_help x, 0) -#define fm__tuple_tag_or_0_(...) fm__tuple_tag_or_0_choose(__VA_ARGS__) -#define fm__tuple_tag_or_0_choose(a,x,...) x -#define fm__tuple_tag_or_0_help(tag, ...) , tag - -#define fm_dispatch_tag_or_0(prefix, x) \ - fm_cat(prefix, fm_tuple_tag_or_0(x)) - /****************************************/ // Iteration @@ -160,20 +164,18 @@ // recursion handle : delay macro expansion to next recursion iteration #define fm_recurs(id) id fm_empty fm_empty() () -#define fm_recurs2(a,b) fm_cat fm_empty fm_empty() () (a,b) +#define fm_recurs2(a,b) fm_cat fm_empty() (a,b) #define fm_defer(id) id fm_empty() #define fm_foreach_join(join, macro, ...) \ - fm_foreach_join_(fm_empty, join, macro, __VA_ARGS__) -#define fm_foreach_join_(join1, join2, macro, ...) \ - fm_cat(fm_foreach_join_, fm_va_01n(__VA_ARGS__))(join1, join2, macro, __VA_ARGS__) + fm_cat(fm_foreach_join_, fm_va_01n(__VA_ARGS__))(fm_empty, join, macro, __VA_ARGS__) #define fm_foreach_join_0(join1, join2, macro, ...) #define fm_foreach_join_1(join1, join2, macro, x) \ join1() macro(x) #define fm_foreach_join_n(join1, join2, macro, x, y, ...) \ join1() macro(x) \ join2() macro(y) \ - fm_recurs2(fm_, foreach_join_) (join2, join2, macro, __VA_ARGS__) + fm_recurs2(fm_foreach_join_, fm_va_01n(__VA_ARGS__))(join2, join2, macro, __VA_ARGS__) #define fm_foreach(macro, ...) \ fm_foreach_join(fm_empty, macro, __VA_ARGS__) @@ -181,16 +183,14 @@ fm_foreach_join(fm_comma, macro, __VA_ARGS__) #define fm_foreach_arg_join(join, macro, arg, ...) \ - fm_foreach_arg_join_(fm_empty, join, macro, arg, __VA_ARGS__) -#define fm_foreach_arg_join_(join1, join2, macro, arg, ...) \ - fm_cat(fm_foreach_arg_join_, fm_va_01n(__VA_ARGS__))(join1, join2, macro, arg, __VA_ARGS__) + fm_cat(fm_foreach_arg_join_, fm_va_01n(__VA_ARGS__))(fm_empty, join, macro, arg, __VA_ARGS__) #define fm_foreach_arg_join_0(join1, join2, macro, ...) #define fm_foreach_arg_join_1(join1, join2, macro, arg, x) \ join1() macro(arg, x) #define fm_foreach_arg_join_n(join1, join2, macro, arg, x, y, ...) \ join1() macro(arg, x) \ join2() macro(arg, y) \ - fm_recurs2(fm_, foreach_arg_join_) (join2, join2, macro, arg, __VA_ARGS__) + fm_recurs2(fm_foreach_arg_join_, fm_va_01n(__VA_ARGS__))(join1, join2, macro, arg, __VA_ARGS__) #define fm_foreach_arg(macro, arg, ...) \ fm_foreach_arg_join(fm_empty, macro, arg, __VA_ARGS__) @@ -198,16 +198,14 @@ fm_foreach_arg_join(fm_comma, macro, arg, __VA_ARGS__) #define fm_foreach_tuple_join(join, macro, ...) \ - fm_foreach_tuple_join_(fm_empty, join, macro, __VA_ARGS__) -#define fm_foreach_tuple_join_(join1, join2, macro, ...) \ - fm_cat(fm_foreach_tuple_join_, fm_va_01n(__VA_ARGS__))(join1, join2, macro, __VA_ARGS__) + fm_cat(fm_foreach_tuple_join_, fm_va_01n(__VA_ARGS__))(fm_empty, join, macro, __VA_ARGS__) #define fm_foreach_tuple_join_0(join1, join2, macro, ...) #define fm_foreach_tuple_join_1(join1, join2, macro, x) \ join1() macro x #define fm_foreach_tuple_join_n(join1, join2, macro, x, y, ...) \ join1() macro x \ join2() macro y \ - fm_recurs2(fm_, foreach_tuple_join_) (join2, join2, macro, __VA_ARGS__) + fm_recurs2(fm_foreach_tuple_join_, fm_va_01n(__VA_ARGS__))(join2, join2, macro, __VA_ARGS__) #define fm_foreach_tuple(macro, ...) \ fm_foreach_tuple_join(fm_empty, macro, __VA_ARGS__) @@ -215,16 +213,14 @@ fm_foreach_tuple_join(fm_comma, macro, __VA_ARGS__) #define fm_foreach_tuple_arg_join(join, macro, arg, ...) \ - fm_foreach_tuple_arg_join_(fm_empty, join, macro, arg, __VA_ARGS__) -#define fm_foreach_tuple_arg_join_(join1, join2, macro, arg, ...) \ - fm_cat(fm_foreach_tuple_arg_join_, fm_va_01n(__VA_ARGS__))(join1, join2, macro, arg, __VA_ARGS__) + fm_cat(fm_foreach_tuple_arg_join_, fm_va_01n(__VA_ARGS__))(fm_empty, join, macro, arg, __VA_ARGS__) #define fm_foreach_tuple_arg_join_0(join1, join2, macro, ...) #define fm_foreach_tuple_arg_join_1(join1, join2, macro, arg, x) \ join1() fm_apply(macro, arg, fm_expand x) #define fm_foreach_tuple_arg_join_n(join1, join2, macro, arg, x, y, ...) \ join1() fm_apply(macro, arg, fm_expand x) \ join2() fm_apply(macro, arg, fm_expand y) \ - fm_recurs2(fm_, foreach_tuple_arg_join_) (join2, join2, macro, arg, __VA_ARGS__) + fm_recurs2(fm_foreach_tuple_arg_join_, fm_va_01n(__VA_ARGS__))(join1, join2, macro, arg, __VA_ARGS__) #define fm_foreach_tuple_arg(macro, arg, ...) \ fm_foreach_tuple_arg_join(fm_empty, macro, arg, __VA_ARGS__) diff --git a/src/fu_util/fo_obj.h b/src/fu_util/fo_obj.h index 70d4ee6b9..6ad423dc6 100644 --- a/src/fu_util/fo_obj.h +++ b/src/fu_util/fo_obj.h @@ -7,7 +7,6 @@ typedef void* fobj_t; #include -#include /* * Pointer to "object*. diff --git a/src/fu_util/ft_array.inc.h b/src/fu_util/ft_array.inc.h index 57d7cad42..847a6393d 100644 --- a/src/fu_util/ft_array.inc.h +++ b/src/fu_util/ft_array.inc.h @@ -1,5 +1,7 @@ /* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ -#include +#ifndef FU_UTIL_H +#error "ft_util.h should be included" +#endif /* * Accepts 2 macroses: @@ -176,7 +178,12 @@ #define ft_array_walk fm_cat(ft_array_pref, _walk) #define ft_array_walk_r fm_cat(ft_array_pref, _walk_r) -#define HUGE_SIZE ((uint64_t)UINT_MAX << 16) +#if __SIZEOF_SIZE_T__ < 8 +#define HUGE_SIZE ((size_t)UINT_MAX >> 2) +#else +#define HUGE_SIZE ((size_t)UINT_MAX << 16) +#endif + #ifndef NDEBUG /* try to catch uninitialized vars */ #define ft_slice_invariants(slc) \ diff --git a/src/fu_util/ft_search.inc.h b/src/fu_util/ft_search.inc.h index b567e11bf..149874cd6 100644 --- a/src/fu_util/ft_search.inc.h +++ b/src/fu_util/ft_search.inc.h @@ -1,3 +1,8 @@ +/* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ +#ifndef FU_UTIL_H +#error "ft_util.h should be included" +#endif + /* * Sort template. * Accepts four macrosses: @@ -39,8 +44,6 @@ * */ -#include - #define ft_func_bsearch fm_cat(ft_bsearch_, FT_SEARCH) #define ft_func_bsearch_r fm_cat3(ft_bsearch_, FT_SEARCH, _r) #define ft_func_search fm_cat(ft_search_, FT_SEARCH) diff --git a/src/fu_util/ft_util.h b/src/fu_util/ft_util.h index 56a0d05d2..084eabf9b 100644 --- a/src/fu_util/ft_util.h +++ b/src/fu_util/ft_util.h @@ -1,24 +1,29 @@ /* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ #ifndef FU_UTIL_H -#define FU_UTIL_H +#define FU_UTIL_H 1 #include #include +#include #include #include #include +#include +#include /* trick to find ssize_t even on windows and strict ansi mode */ #if defined(_MSC_VER) #include typedef SSIZE_T ssize_t; -#else -#include +#define SSIZE_MAX ((ssize_t)((SIZE_MAX) >> 1)) + +#if !defined(WIN32) && defined(_WIN32) +#define WIN32 _WIN32 +#endif + #endif #include #include #include -#include - #ifdef __GNUC__ #define ft_gcc_const __attribute__((const)) @@ -29,7 +34,7 @@ typedef SSIZE_T ssize_t; #define ft_gcc_malloc(free, idx) __attribute__((malloc)) #endif #define ft_unused __attribute__((unused)) -#define ft_gnu_printf(fmt, arg) __attribute__((format(printf,fmt,arg))) +#define ft_gnu_printf(fmt, arg) __attribute__((format(gnu_printf,fmt,arg))) #define ft_likely(x) __builtin_expect(!!(x), 1) #define ft_unlikely(x) __builtin_expect(!!(x), 0) #define ft_always_inline __attribute__((always_inline)) @@ -103,6 +108,7 @@ typedef void ft_gnu_printf(4, 0) (*ft_log_hook_t)(enum FT_LOG_LEVEL, /* * Initialize logging in main executable file. * Pass custom hook or NULL. + * In MinGW if built with libbacktrace, pass executable path (argv[0]). */ #define ft_init_log(hook) ft__init_log(hook, __FILE__) @@ -135,7 +141,7 @@ const char* ft__truncate_log_filename(const char *file); #define ft_dbg_enabled() ft__dbg_enabled() #define ft_dbg_assert(x, ...) ft__dbg_assert(x, #x, __VA_ARGS__) -#define ft_assert(x, ...) ft__assert(x, #x, __VA_ARGS__) +#define ft_assert(x, ...) ft__assert(x, #x, ##__VA_ARGS__) #define ft_assyscall(syscall, ...) ft__assyscall(syscall, fm_uniq(res), __VA_ARGS__) /* threadsafe strerror */ @@ -305,13 +311,14 @@ typedef struct ft_bytes_t { } ft_bytes_t; ft_inline ft_bytes_t ft_bytes(void* ptr, size_t len) { - return (ft_bytes_t){.ptr = ptr, .len = len}; + return (ft_bytes_t){.ptr = (char*)ptr, .len = len}; } ft_inline void ft_bytes_consume(ft_bytes_t *bytes, size_t cut); ft_inline void ft_bytes_move(ft_bytes_t *dest, ft_bytes_t *src); // String utils +extern size_t ft_strlcpy(char *dest, const char* src, size_t dest_size); /* * Concat strings regarding destination buffer size. * Note: if dest already full and doesn't contain \0n character, then fatal log is issued. @@ -411,7 +418,7 @@ extern bool ft_strbuf_vcatf (ft_strbuf_t *buf, const char *fmt, va_list * Use it if format string comes from user. */ ft_gnu_printf(3, 0) -extern bool ft_strbuf_vcatf_err (ft_strbuf_t *buf, bool err[static 1], +extern bool ft_strbuf_vcatf_err (ft_strbuf_t *buf, bool err[1], const char *fmt, va_list args); /* * Returns string which points into the buffer. diff --git a/src/fu_util/impl/fo_impl.c b/src/fu_util/impl/fo_impl.c index bbc49ab7f..63fa372fb 100644 --- a/src/fu_util/impl/fo_impl.c +++ b/src/fu_util/impl/fo_impl.c @@ -5,13 +5,9 @@ #include #include -#ifdef WIN32 -#define __thread __declspec(thread) -#endif -#include - #include -#include + +#include /* * We limits total number of methods, klasses and method implementations. @@ -650,7 +646,7 @@ fobjStr* fobj_newstr(ft_str_t s, enum FOBJ_STR_ALLOC ownership) { fobjStr *str; #if __SIZEOF_POINTER__ < 8 - ft_assert(size < (1<<30)-2); + ft_assert(s.len < (1<<30)-2); #else ft_assert(s.len < UINT32_MAX-2); #endif @@ -871,13 +867,13 @@ fobj_format_int(ft_strbuf_t *buf, uint64_t i, bool _signed, const char *fmt) { /* now add real suitable format */ switch (base) { - case 'x': strcat(tfmt + fmtlen, PRIx64); break; - case 'X': strcat(tfmt + fmtlen, PRIX64); break; - case 'o': strcat(tfmt + fmtlen, PRIo64); break; - case 'u': strcat(tfmt + fmtlen, PRIu64); break; - case 'd': strcat(tfmt + fmtlen, PRId64); break; + case 'x': ft_strlcat(tfmt, PRIx64, sizeof(tfmt)); break; + case 'X': ft_strlcat(tfmt, PRIX64, sizeof(tfmt)); break; + case 'o': ft_strlcat(tfmt, PRIo64, sizeof(tfmt)); break; + case 'u': ft_strlcat(tfmt, PRIu64, sizeof(tfmt)); break; + case 'd': ft_strlcat(tfmt, PRId64, sizeof(tfmt)); break; default: - case 'i': strcat(tfmt + fmtlen, PRIi64); break; + case 'i': ft_strlcat(tfmt, PRIi64, sizeof(tfmt)); break; } switch (base) { @@ -1082,11 +1078,11 @@ fobj__format_errmsg(const char* msg, fobj_err_kv_t *kvs) { "ident is too long in message \"%s\"", msg); ft_assert(formatdelim == NULL || closebrace - formatdelim <= 31, "format is too long in message \"%s\"", msg); - strncpy(ident, cur, identlen); + memcpy(ident, cur, identlen); ident[identlen] = 0; formatlen = formatdelim ? closebrace - (formatdelim+1) : 0; if (formatlen > 0) { - strncpy(format, formatdelim + 1, formatlen); + memcpy(format, formatdelim + 1, formatlen); } format[formatlen] = 0; kv = kvs; @@ -1293,11 +1289,11 @@ fobj_printkv(const char *fmt, ft_slc_fokv_t kvs) { "ident is too long in format \"%s\"", fmt); ft_assert(formatdelim == NULL || closebrace - formatdelim <= 31, "format is too long in format \"%s\"", fmt); - strncpy(ident, cur, identlen); + memcpy(ident, cur, identlen); ident[identlen] = 0; formatlen = formatdelim ? closebrace - (formatdelim+1) : 0; if (formatlen > 0) { - strncpy(format, formatdelim + 1, formatlen); + memcpy(format, formatdelim + 1, formatlen); } format[formatlen] = 0; i = ft_search_fokv(kvs.ptr, kvs.len, ident, fobj_fokv_cmpc); diff --git a/src/fu_util/impl/fo_impl2.h b/src/fu_util/impl/fo_impl2.h index 916714997..1cac933a0 100644 --- a/src/fu_util/impl/fo_impl2.h +++ b/src/fu_util/impl/fo_impl2.h @@ -2,10 +2,6 @@ #ifndef FOBJ_OBJ_PRIV2_H #define FOBJ_OBJ_PRIV2_H -#include -#include -#include - enum fobjStrType { FOBJ_STR_SMALL = 1, FOBJ_STR_UNOWNED, @@ -57,6 +53,7 @@ fobj_getstr(fobjStr *str) { return ft_str(str->ptr.ptr, str->ptr.len); default: ft_log(FT_FATAL, "Unknown fobj_str type %d", str->type); + return ft_str(NULL, 0); } } diff --git a/src/fu_util/impl/ft_impl.c b/src/fu_util/impl/ft_impl.c index 097171e86..1897e6bec 100644 --- a/src/fu_util/impl/ft_impl.c +++ b/src/fu_util/impl/ft_impl.c @@ -1,25 +1,35 @@ /* vim: set expandtab autoindent cindent ts=4 sw=4 sts=4 */ -#include +#include + #include -#include -#include #include +#include +#if !defined(WIN32) || defined(__MINGW64__) || defined(__MINGW32__) #include #include -#include +#else +#define WIN32_LEAN_AND_MEAN + +#include +#include +#include +#undef small +#include +#include +#include +#undef near +#endif + #ifdef HAVE_LIBBACKTRACE #include -#else +#if defined(__MINGW32__) || defined(__MINGW64__) +#include +#endif +#elif HAVE_EXECINFO_H #include #endif -#ifdef WIN32 -#define __thread __declspec(thread) -#else #include -#endif - -#include #define FT_LOG_MAX_FILES (1<<12) @@ -100,12 +110,22 @@ ft_strlcat(char *dest, const char* src, size_t dest_size) { ft_assert(dest_null, "destination has no zero byte"); if (dest_len < dest_size-1) { size_t cpy_len = dest_size - dest_len - 1; - strncpy(dest+dest_len, src, cpy_len); + cpy_len = ft_min(cpy_len, strlen(src)); + memcpy(dest+dest_len, src, cpy_len); dest[dest_len + cpy_len] = '\0'; } return dest_len + strlen(src); } +size_t +ft_strlcpy(char *dest, const char* src, size_t dest_size) { + size_t cpy_len = dest_size - 1; + cpy_len = ft_min(cpy_len, strlen(src)); + memcpy(dest, src, cpy_len); + dest[cpy_len] = '\0'; + return strlen(src); +} + ft_str_t ft_vasprintf(const char *fmt, va_list args) { ft_strbuf_t buf = ft_strbuf_zero(); @@ -302,9 +322,23 @@ ft__base_log_filename(const char *file) { static struct backtrace_state * volatile ft_btstate = NULL; static pthread_once_t ft_btstate_once = PTHREAD_ONCE_INIT; + +static void +ft_backtrace_err(void *data, const char *msg, int errnum) +{ + fprintf(stderr, "ft_backtrace_err %s %d\n", msg, errnum); +} + static void ft_backtrace_init(void) { - __atomic_store_n(&ft_btstate, backtrace_create_state(NULL, 0, NULL, NULL), + const char *app = NULL; +#if defined(__MINGW32__) || defined(__MINGW64__) + static char appbuf[2048] = {0}; + /* 2048 should be enough, don't check error */ + GetModuleFileNameA(0, appbuf, sizeof(appbuf)-1); + app = appbuf; +#endif + __atomic_store_n(&ft_btstate, backtrace_create_state(app, 1, ft_backtrace_err, NULL), __ATOMIC_RELEASE); } @@ -315,9 +349,9 @@ ft_backtrace_add(void *data, uintptr_t pc, struct ft_strbuf_t *buf = data; ssize_t sz; if (filename == NULL) - return 1; - return ft_strbuf_catf(buf, "\n%s:%-4d %s", - ft__truncate_log_filename(filename), lineno, function); + return 0; + return !ft_strbuf_catf(buf, "\n\t%s:%-4d\t%s", + ft__truncate_log_filename(filename), lineno, function ? function : "(unknown)"); } #endif @@ -355,9 +389,9 @@ ft_default_log(enum FT_LOG_LEVEL level, ft_source_position_t srcpos, #ifdef HAVE_LIBBACKTRACE if (__atomic_load_n(&ft_btstate, __ATOMIC_ACQUIRE) == NULL) pthread_once(&ft_btstate_once, ft_backtrace_init); - - backtrace_full(ft_btstate, 1, ft_backtrace_add, NULL, &buf); -#else + if (ft_btstate) + backtrace_full(ft_btstate, 0, ft_backtrace_add, NULL, &buf); +#elif defined(HAVE_EXECINFO_H) void *backtr[32] = {0}; char **syms = NULL; int i, n; @@ -414,7 +448,15 @@ ft__log_fatal(ft_source_position_t srcpos, const char* error, const char* ft__strerror(int eno, char *buf, size_t len) { -#if !_GNU_SOURCE && (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) +#ifndef HAVE_STRERROR_R + char *sbuf = strerror(eno); + + if (sbuf == NULL) /* can this still happen anywhere? */ + return NULL; + /* To minimize thread-unsafety hazard, copy into caller's buffer */ + ft_strlcpy(buf, sbuf, len); + return buf; +#elif !_GNU_SOURCE && (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) int saveno = errno; int e = strerror_r(eno, buf, len); if (e != 0) { diff --git a/src/fu_util/test/CMakeLists.txt b/src/fu_util/test/CMakeLists.txt index 06f86effc..05eea86c6 100644 --- a/src/fu_util/test/CMakeLists.txt +++ b/src/fu_util/test/CMakeLists.txt @@ -1,5 +1,10 @@ cmake_minimum_required(VERSION 3.11) +add_executable(fm fm.c) +add_executable(fm1 fm.c) + +target_compile_options(fm1 PRIVATE -DFM_USE_STRICT=1) + add_executable(array array.c) target_link_libraries(array fu_utils) @@ -20,6 +25,8 @@ target_link_libraries(obj1 fu_utils) enable_testing() +add_test(NAME fm COMMAND fm) +add_test(NAME fm1 COMMAND fm1) add_test(NAME array COMMAND array) add_test(NAME bsearch COMMAND bsearch) add_test(NAME fuprintf COMMAND fuprintf) diff --git a/src/fu_util/test/obj1.c b/src/fu_util/test/obj1.c index db5f9b0b0..faa7aafa5 100644 --- a/src/fu_util/test/obj1.c +++ b/src/fu_util/test/obj1.c @@ -285,6 +285,7 @@ int main(int argc, char** argv) { ft_assert(fobj_streq_c(strf, "Some scary things cost > $$12.4800 $$"), "String is '%s'", $tostr(strf)); + ft_log(FT_ERROR, "and try backtrace"); logf("BEFORE EXIT"); } diff --git a/src/fu_util/test/qsort/qsort.inc.c b/src/fu_util/test/qsort/qsort.inc.c index c801ae52a..2a53ae93f 100644 --- a/src/fu_util/test/qsort/qsort.inc.c +++ b/src/fu_util/test/qsort/qsort.inc.c @@ -20,7 +20,6 @@ Engineering a sort function; Jon Bentley and M. Douglas McIlroy; Software - Practice and Experience; Vol. 23 (11), 1249-1265, 1993. */ -#include #include #include #include diff --git a/src/init.c b/src/init.c index 41ee2e3c9..511256aa3 100644 --- a/src/init.c +++ b/src/init.c @@ -11,7 +11,6 @@ #include "pg_probackup.h" #include -#include /* * Initialize backup catalog. diff --git a/src/merge.c b/src/merge.c index 03698e92d..f64b72611 100644 --- a/src/merge.c +++ b/src/merge.c @@ -9,7 +9,6 @@ #include "pg_probackup.h" -#include #include #include "utils/thread.h" @@ -958,7 +957,7 @@ merge_files(void *arg) if (S_ISDIR(dest_file->mode)) goto done; - elog(progress ? INFO : LOG, "Progress: (%d/%lu). Merging file \"%s\"", + elog(progress ? INFO : LOG, "Progress: (%d/%zu). Merging file \"%s\"", i + 1, n_files, dest_file->rel_path); if (dest_file->is_datafile && !dest_file->is_cfs) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b7308405c..dda5cf65a 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -47,8 +47,6 @@ #include "streamutil.h" #include "utils/file.h" -#include - #include "utils/configuration.h" #include "utils/thread.h" #include @@ -75,7 +73,6 @@ bool no_color = false; bool show_color = true; bool is_archive_cmd = false; pid_t my_pid = 0; -__thread int my_thread_num = 1; bool progress = false; bool no_sync = false; time_t start_time = 0; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7ce455459..4d68789a5 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -14,7 +14,6 @@ #include "postgres_fe.h" #include "libpq-fe.h" -#include "libpq-int.h" #include "access/xlog_internal.h" #include "utils/pg_crc.h" @@ -44,11 +43,8 @@ #include "pg_probackup_state.h" - -#ifdef WIN32 -#define __thread __declspec(thread) -#else -#include +#if defined(WIN32) && !(defined(_UCRT) && defined(__MINGW64__)) +#error Windows port requires compilation in MinGW64 UCRT environment #endif /* Wrap the code that we're going to delete after refactoring in this define*/ @@ -207,28 +203,6 @@ typedef enum ForkName ptrack } ForkName; -#define INIT_FILE_CRC32(use_crc32c, crc) \ -do { \ - if (use_crc32c) \ - INIT_CRC32C(crc); \ - else \ - INIT_TRADITIONAL_CRC32(crc); \ -} while (0) -#define COMP_FILE_CRC32(use_crc32c, crc, data, len) \ -do { \ - if (use_crc32c) \ - COMP_CRC32C((crc), (data), (len)); \ - else \ - COMP_TRADITIONAL_CRC32(crc, data, len); \ -} while (0) -#define FIN_FILE_CRC32(use_crc32c, crc) \ -do { \ - if (use_crc32c) \ - FIN_CRC32C(crc); \ - else \ - FIN_TRADITIONAL_CRC32(crc); \ -} while (0) - #define pg_off_t unsigned long long @@ -760,7 +734,6 @@ typedef struct StopBackupCallbackParams /* common options */ extern pid_t my_pid; -extern __thread int my_thread_num; extern int num_threads; extern bool stream_wal; extern bool show_color; @@ -1045,8 +1018,11 @@ extern pgFile *pgFileNew(const char *path, const char *rel_path, extern pgFile *pgFileInit(const char *rel_path); extern void pgFileFree(void *file); -extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok); -extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok); +extern pg_crc32 pgFileGetCRC32C(const char *file_path, bool missing_ok); +#if PG_VERSION_NUM < 120000 +extern pg_crc32 pgFileGetCRC32(const char *file_path, bool missing_ok); +#endif +extern pg_crc32 pgFileGetCRC32Cgz(const char *file_path, bool missing_ok); extern int pgFileMapComparePath(const void *f1, const void *f2); extern int pgFileCompareName(const void *f1, const void *f2); diff --git a/src/restore.c b/src/restore.c index 28a79f1ed..9113a15da 100644 --- a/src/restore.c +++ b/src/restore.c @@ -12,7 +12,6 @@ #include "access/timeline.h" -#include #include #include "utils/thread.h" @@ -1105,14 +1104,14 @@ static void * restore_files(void *arg) { int i; - uint64 n_files; + size_t n_files; char to_fullpath[MAXPGPATH]; FILE *out = NULL; char *out_buf = pgut_malloc(STDIO_BUFSIZE); restore_files_arg *arguments = (restore_files_arg *) arg; - n_files = (unsigned long) parray_num(arguments->dest_files); + n_files = parray_num(arguments->dest_files); for (i = 0; i < parray_num(arguments->dest_files); i++) { @@ -1133,7 +1132,7 @@ restore_files(void *arg) if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during restore"); - elog(progress ? INFO : LOG, "Progress: (%d/%lu). Restore file \"%s\"", + elog(progress ? INFO : LOG, "Progress: (%d/%zu). Restore file \"%s\"", i + 1, n_files, dest_file->rel_path); /* Only files from pgdata can be skipped by partial restore */ @@ -2010,7 +2009,6 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, { int i; int j; -// pg_crc32 crc; parray *database_map = NULL; parray *dbOid_exclude_list = NULL; pgFile *database_map_file = NULL; @@ -2040,13 +2038,6 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); - /* check database_map CRC */ -// crc = pgFileGetCRC(database_map_path, true, true, NULL, FIO_LOCAL_HOST); -// -// if (crc != database_map_file->crc) -// elog(ERROR, "Invalid CRC of backup file \"%s\" : %X. Expected %X", -// database_map_file->path, crc, database_map_file->crc); - /* get database_map from file */ database_map = read_database_map(backup); @@ -2174,8 +2165,8 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, } else if (pid > 1) /* postmaster is up */ { - elog(WARNING, "Postmaster with pid %u is running in destination directory \"%s\"", - pid, pgdata); + elog(WARNING, "Postmaster with pid %llu is running in destination directory \"%s\"", + (long long)pid, pgdata); success = false; postmaster_is_up = true; } @@ -2198,9 +2189,9 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, if (system_id_pgdata == instance_config.system_identifier) system_id_match = true; else - elog(WARNING, "Backup catalog was initialized for system id %lu, " - "but destination directory system id is %lu", - system_identifier, system_id_pgdata); + elog(WARNING, "Backup catalog was initialized for system id %llu, " + "but destination directory system id is %llu", + (long long)system_identifier, (long long)system_id_pgdata); /* * TODO: maybe there should be some other signs, pointing to pg_control diff --git a/src/show.c b/src/show.c index db8a9e225..5440e28a2 100644 --- a/src/show.c +++ b/src/show.c @@ -13,7 +13,6 @@ #include #include #include -#include #include "utils/json.h" @@ -911,7 +910,7 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, cur++; /* N files */ - snprintf(row->n_segments, lengthof(row->n_segments), "%lu", + snprintf(row->n_segments, lengthof(row->n_segments), "%zu", tlinfo->n_xlog_files); widths[cur] = Max(widths[cur], strlen(row->n_segments)); cur++; @@ -931,7 +930,7 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, cur++; /* N backups */ - snprintf(row->n_backups, lengthof(row->n_backups), "%lu", + snprintf(row->n_backups, lengthof(row->n_backups), "%zu", tlinfo->backups?parray_num(tlinfo->backups):0); widths[cur] = Max(widths[cur], strlen(row->n_backups)); cur++; @@ -1087,10 +1086,10 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, json_add_value(buf, "max-segno", tmp_buf, json_level, true); json_add_key(buf, "n-segments", json_level); - appendPQExpBuffer(buf, "%lu", tlinfo->n_xlog_files); + appendPQExpBuffer(buf, "%zu", tlinfo->n_xlog_files); json_add_key(buf, "size", json_level); - appendPQExpBuffer(buf, "%lu", tlinfo->size); + appendPQExpBuffer(buf, "%zu", tlinfo->size); json_add_key(buf, "zratio", json_level); diff --git a/src/stream.c b/src/stream.c index b10eb7308..e2e016f4d 100644 --- a/src/stream.c +++ b/src/stream.c @@ -2,7 +2,7 @@ * * stream.c: pg_probackup specific code for WAL streaming * - * Portions Copyright (c) 2015-2022, Postgres Professional + * Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -689,7 +689,7 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos if (existing_file) { if (do_crc) - (*existing_file)->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + (*existing_file)->crc = pgFileGetCRC32C(wal_segment_fullpath, false); (*existing_file)->write_size = xlog_seg_size; (*existing_file)->uncompressed_size = xlog_seg_size; @@ -697,7 +697,7 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos } if (do_crc) - file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + file->crc = pgFileGetCRC32C(wal_segment_fullpath, false); /* Should we recheck it using stat? */ file->write_size = xlog_seg_size; @@ -728,7 +728,7 @@ add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir) /* calculate crc */ if (do_crc) - file->crc = pgFileGetCRC(fullpath, true, false); + file->crc = pgFileGetCRC32C(fullpath, false); file->write_size = file->size; file->uncompressed_size = file->size; diff --git a/src/util.c b/src/util.c index b58d88f96..e16241a70 100644 --- a/src/util.c +++ b/src/util.c @@ -304,7 +304,7 @@ get_pgcontrol_checksum(const char *pgdata_path) /* First fetch file... */ buffer = slurpFile(FIO_BACKUP_HOST, pgdata_path, XLOG_CONTROL_FILE, &size, false); - + elog(WARNING, "checking %s", pgdata_path); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); diff --git a/src/utils/file.c b/src/utils/file.c index 86977a19a..9f72c7dc4 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2,8 +2,7 @@ #include #include "pg_probackup.h" -/* sys/stat.h must be included after pg_probackup.h (see problems with compilation for windows described in PGPRO-5750) */ -#include +#include #include "file.h" #include "storage/checksum.h" @@ -78,8 +77,8 @@ typedef struct #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) #if defined(WIN32) -#undef open(a, b, c) -#undef fopen(a, b) +#undef open +#undef fopen #endif void @@ -139,43 +138,6 @@ fio_is_remote_fd(int fd) return (fd & FIO_PIPE_MARKER) != 0; } -#ifdef WIN32 - -#undef stat - -/* - * The stat() function in win32 is not guaranteed to update the st_size - * field when run. So we define our own version that uses the Win32 API - * to update this field. - */ -static int -fio_safestat(const char *path, struct stat *buf) -{ - int r; - WIN32_FILE_ATTRIBUTE_DATA attr; - - r = stat(path, buf); - if (r < 0) - return r; - - if (!GetFileAttributesEx(path, GetFileExInfoStandard, &attr)) - { - errno = ENOENT; - return -1; - } - - /* - * XXX no support for large files here, but we don't do that in general on - * Win32 yet. - */ - buf->st_size = attr.nFileSizeLow; - - return 0; -} - -#define stat(x, y) fio_safestat(x, y) -#endif /* WIN32 */ - #ifdef WIN32 /* TODO: use real pread on Linux */ static ssize_t @@ -194,7 +156,7 @@ remove_file_or_dir(const char* path) { int rc = remove(path); - if (rc < 0 && errno == EACCESS) + if (rc < 0 && errno == EACCES) rc = rmdir(path); return rc; } @@ -1428,9 +1390,9 @@ fio_get_crc32(fio_location location, const char *file_path, bool decompress) else { if (decompress) - return pgFileGetCRCgz(file_path, true, true); + return pgFileGetCRC32Cgz(file_path, true); else - return pgFileGetCRC(file_path, true, true); + return pgFileGetCRC32C(file_path, true); } } @@ -2082,7 +2044,7 @@ fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, Assert(hdr.size <= sizeof(buf)); IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + COMP_CRC32C(file->crc, buf, hdr.size); /* lazily open backup file */ if (!out) @@ -2203,10 +2165,10 @@ fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, strerror(errno)); - elog(VERBOSE, "ftruncate file \"%s\" to size %lu", + elog(VERBOSE, "ftruncate file \"%s\" to size %zu", to_fullpath, file->size); if (fio_ftruncate(out, file->size) == -1) - elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", + elog(ERROR, "Cannot ftruncate file \"%s\" to size %zu: %s", to_fullpath, file->size, strerror(errno)); if (!fio_is_remote_file(out)) @@ -2270,8 +2232,6 @@ fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, Assert(hdr.size <= sizeof(buf)); IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); - if (fio_fseek(out, blknum * BLCKSZ) < 0) { elog(ERROR, "Cannot seek block %u of \"%s\": %s", @@ -2635,7 +2595,7 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, if (file) { file->read_size += hdr.size; - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + COMP_CRC32C(file->crc, buf, hdr.size); } } else @@ -3100,6 +3060,7 @@ local_check_postmaster(const char *pgdata) { FILE *fp; pid_t pid; + long long lpid; char pid_file[MAXPGPATH]; join_path_components(pid_file, pgdata, "postmaster.pid"); @@ -3115,7 +3076,11 @@ local_check_postmaster(const char *pgdata) pid_file, strerror(errno)); } - if (fscanf(fp, "%i", &pid) != 1) + if (fscanf(fp, "%lli", &lpid) == 1) + { + pid = lpid; + } + else { /* something is wrong with the file content */ pid = 1; @@ -3129,8 +3094,8 @@ local_check_postmaster(const char *pgdata) if (errno == ESRCH) pid = 0; else - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - pid, strerror(errno)); + elog(ERROR, "Failed to send signal 0 to a process %lld: %s", + (long long)pid, strerror(errno)); } } @@ -3366,9 +3331,9 @@ fio_communicate(int in, int out) case FIO_GET_CRC32: /* calculate crc32 for a file */ if (hdr.arg == 1) - crc = pgFileGetCRCgz(buf, true, true); + crc = pgFileGetCRC32Cgz(buf, true); else - crc = pgFileGetCRC(buf, true, true); + crc = pgFileGetCRC32C(buf, true); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); break; case FIO_GET_CHECKSUM_MAP: @@ -3606,9 +3571,9 @@ pioLocalDrive_pioGetCRC32(VSelf, path_t path, bool compressed, err_i *err) elog(VERBOSE, "Local Drive calculate crc32 for '%s', compressed=%d", path, compressed); if (compressed) - return pgFileGetCRCgz(path, true, true); + return pgFileGetCRC32Cgz(path, true); else - return pgFileGetCRC(path, true, true); + return pgFileGetCRC32C(path, true); } static bool diff --git a/src/utils/file.h b/src/utils/file.h index 7fd1e7919..79e86ee20 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -3,7 +3,9 @@ #include "storage/bufpage.h" #include +#ifndef WIN32 #include +#endif #include #ifdef HAVE_LIBZ @@ -223,11 +225,13 @@ fobj_iface(pioWriteFlush); fobj_iface(pioWriteCloser); fobj_iface(pioReadCloser); +typedef struct stat stat_t; + // Drive #define mth__pioOpen pioFile_i, (path_t, path), (int, flags), \ (int, permissions), (err_i *, err) #define mth__pioOpen__optional() (permissions, FILE_PERMISSION) -#define mth__pioStat struct stat, (path_t, path), (bool, follow_symlink), \ +#define mth__pioStat stat_t, (path_t, path), (bool, follow_symlink), \ (err_i *, err) #define mth__pioRemove err_i, (path_t, path), (bool, missing_ok) #define mth__pioRename err_i, (path_t, old_path), (path_t, new_path) diff --git a/src/utils/logger.c b/src/utils/logger.c index e58802e28..57b96e020 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -343,8 +343,8 @@ elog_internal(int elevel, bool file_only, const char *message) if (format_file == JSON || format_console == JSON) { - snprintf(str_pid_json, sizeof(str_pid_json), "%d", my_pid); - snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num); + snprintf(str_pid_json, sizeof(str_pid_json), "%lld", (long long)my_pid); + snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num()); initPQExpBuffer(&show_buf); json_add_min(buf_json, JT_BEGIN_OBJECT); @@ -357,7 +357,7 @@ elog_internal(int elevel, bool file_only, const char *message) json_add_min(buf_json, JT_END_OBJECT); } - snprintf(str_pid, sizeof(str_pid), "[%d]:", my_pid); + snprintf(str_pid, sizeof(str_pid), "[%lld]:", (long long)my_pid); /* * Write message to log file. @@ -424,7 +424,7 @@ elog_internal(int elevel, bool file_only, const char *message) { char str_thread[64]; /* [Issue #213] fix pgbadger parsing */ - snprintf(str_thread, sizeof(str_thread), "[%d-1]:", my_thread_num); + snprintf(str_thread, sizeof(str_thread), "[%d-1]:", my_thread_num()); fprintf(stderr, "%s ", strfbuf); fprintf(stderr, "%s ", str_pid); @@ -497,8 +497,8 @@ elog_stderr(int elevel, const char *fmt, ...) { strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", localtime(&log_time)); - snprintf(str_pid, sizeof(str_pid), "%d", my_pid); - snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num); + snprintf(str_pid, sizeof(str_pid), "%lld", (long long)my_pid); + snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num()); initPQExpBuffer(&show_buf); json_add_min(buf_json, JT_BEGIN_OBJECT); @@ -811,11 +811,7 @@ logfile_getname(const char *format, time_t timestamp) len = strlen(filename); /* Treat log_filename as a strftime pattern */ -#ifdef WIN32 - if (pg_strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) -#else if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) -#endif elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); return filename; @@ -971,7 +967,7 @@ open_logfile(FILE **file, const char *filename_format) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); - fprintf(control_file, "%ld", timestamp); + fprintf(control_file, "%lld", (long long)timestamp); fclose(control_file); } diff --git a/src/utils/pgut.c b/src/utils/pgut.c index f1b8da0b2..9a7b465ee 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1061,20 +1061,6 @@ init_cancel_handler(void) SetConsoleCtrlHandler(consoleHandler, TRUE); } -int -sleep(unsigned int seconds) -{ - Sleep(seconds * 1000); - return 0; -} - -int -usleep(unsigned int usec) -{ - Sleep((usec + 999) / 1000); /* rounded up */ - return 0; -} - #undef select static int select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout) diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 638259a3c..e85104285 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -3,7 +3,7 @@ * pgut.h * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2021, Postgres Professional + * Portions Copyright (c) 2017-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -11,6 +11,7 @@ #ifndef PGUT_H #define PGUT_H +#include #include "postgres_fe.h" #include "libpq-fe.h" @@ -98,9 +99,4 @@ extern char *pgut_str_strip_trailing_filename(const char *filepath, const char * extern int wait_for_socket(int sock, struct timeval *timeout); extern int wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout); -#ifdef WIN32 -extern int sleep(unsigned int seconds); -extern int usleep(unsigned int usec); -#endif - #endif /* PGUT_H */ diff --git a/src/utils/remote.c b/src/utils/remote.c index 3286052a5..bceccc26a 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -5,12 +5,6 @@ #include #include -#ifdef WIN32 -#define __thread __declspec(thread) -#else -#include -#endif - #include "pg_probackup.h" #include "file.h" @@ -113,14 +107,14 @@ bool launch_agent(void) char cmd[MAX_CMDLINE_LENGTH]; char* ssh_argv[MAX_CMDLINE_OPTIONS]; int ssh_argc; - int outfd[2]; - int infd[2]; - int errfd[2]; + int outfd[2] = {0, 0}; + int infd[2] = {0, 0}; + int errfd[2] = {0, 0}; int agent_version; ssh_argc = 0; #ifdef WIN32 - ssh_argv[ssh_argc++] = PROGRAM_NAME_FULL; + ssh_argv[ssh_argc++] = (char *) PROGRAM_NAME_FULL; ssh_argv[ssh_argc++] = "ssh"; ssh_argc += 2; /* reserve space for pipe descriptors */ #endif @@ -198,7 +192,9 @@ bool launch_agent(void) ssh_argv[2] = psprintf("%d", outfd[0]); ssh_argv[3] = psprintf("%d", infd[1]); { - intptr_t pid = _spawnvp(_P_NOWAIT, ssh_argv[0], ssh_argv); + intptr_t pid = _spawnvp(_P_NOWAIT, + (const char*)ssh_argv[0], + (const char * const *) ssh_argv); if (pid < 0) return false; child_pid = GetProcessId((HANDLE)pid); diff --git a/src/utils/thread.c b/src/utils/thread.c index 1c469bd29..1ad1e772e 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -17,97 +17,17 @@ */ bool thread_interrupted = false; -#ifdef WIN32 -DWORD main_tid = 0; -#else pthread_t main_tid = 0; -#endif -#ifdef WIN32 -#include - -typedef struct win32_pthread -{ - HANDLE handle; - void *(*routine) (void *); - void *arg; - void *result; -} win32_pthread; - -static long mutex_initlock = 0; - -static unsigned __stdcall -win32_pthread_run(void *arg) -{ - win32_pthread *th = (win32_pthread *)arg; - - th->result = th->routine(th->arg); - - return 0; -} - -int -pthread_create(pthread_t *thread, - pthread_attr_t *attr, - void *(*start_routine) (void *), - void *arg) -{ - int save_errno; - win32_pthread *th; - - th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); - th->routine = start_routine; - th->arg = arg; - th->result = NULL; - - th->handle = (HANDLE)_beginthreadex(NULL, 0, win32_pthread_run, th, 0, NULL); - if (th->handle == NULL) - { - save_errno = errno; - free(th); - return save_errno; - } - - *thread = th; - return 0; -} +static __thread int my_thread_num_var = 1; int -pthread_join(pthread_t th, void **thread_return) +my_thread_num(void) { - if (th == NULL || th->handle == NULL) - return errno = EINVAL; - - if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) - { - _dosmaperr(GetLastError()); - return errno; - } - - if (thread_return) - *thread_return = th->result; - - CloseHandle(th->handle); - free(th); - return 0; + return my_thread_num_var; } -#endif /* WIN32 */ - -int -pthread_lock(pthread_mutex_t *mp) +void +set_my_thread_num(int th) { -#ifdef WIN32 - if (*mp == NULL) - { - while (InterlockedExchange(&mutex_initlock, 1) == 1) - /* loop, another thread own the lock */ ; - if (*mp == NULL) - { - if (pthread_mutex_init(mp, NULL)) - return -1; - } - InterlockedExchange(&mutex_initlock, 0); - } -#endif - return pthread_mutex_lock(mp); + my_thread_num_var = th; } diff --git a/src/utils/thread.h b/src/utils/thread.h index 2eaa5fb45..a6c58f70e 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -10,32 +10,18 @@ #ifndef PROBACKUP_THREAD_H #define PROBACKUP_THREAD_H -#ifdef WIN32 -#include "postgres_fe.h" -#include "port/pthread-win32.h" - -/* Use native win32 threads on Windows */ -typedef struct win32_pthread *pthread_t; -typedef int pthread_attr_t; - -#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } -#define PTHREAD_ONCE_INIT false +#if defined(WIN32) && !(defined(__MINGW64__) || defined(__MINGW32__) || defined(HAVE_PTHREAD)) +#error "Windows build supports only 'pthread' threading" +#endif -extern int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); -extern int pthread_join(pthread_t th, void **thread_return); -#else /* Use platform-dependent pthread capability */ #include -#endif - -#ifdef WIN32 -extern DWORD main_tid; -#else extern pthread_t main_tid; -#endif +#define pthread_lock(mp) pthread_mutex_lock(mp) extern bool thread_interrupted; -extern int pthread_lock(pthread_mutex_t *mp); +int my_thread_num(void); +void set_my_thread_num(int); #endif /* PROBACKUP_THREAD_H */ diff --git a/src/validate.c b/src/validate.c index 79a450ac8..84a27adcb 100644 --- a/src/validate.c +++ b/src/validate.c @@ -3,14 +3,13 @@ * validate.c: validate backup files. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ #include "pg_probackup.h" -#include #include #include "utils/thread.h" @@ -313,8 +312,8 @@ pgBackupValidateFiles(void *arg) if (file->write_size != st.st_size) { - elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", - file_fullpath, (unsigned long) st.st_size, file->write_size); + elog(WARNING, "Invalid size of backup file \"%s\" : %lld. Expected %lld", + file_fullpath, (long long) st.st_size, (long long)file->write_size); arguments->corrupted = true; break; } @@ -341,14 +340,22 @@ pgBackupValidateFiles(void *arg) * Starting from 2.0.25 we calculate crc of pg_control differently. */ if (arguments->backup_version >= 20025 && - strcmp(file->name, "pg_control") == 0 && - !file->external_dir_num) + strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0 && + file->external_dir_num == 0) crc = get_pgcontrol_checksum(arguments->base_path); else - crc = pgFileGetCRC(file_fullpath, - arguments->backup_version <= 20021 || - arguments->backup_version >= 20025, - false); +#if PG_VERSION_NUM >= 120000 + { + Assert(arguments->backup_version >= 20025); + crc = pgFileGetCRC32C(file_fullpath, false); + } +#else /* PG_VERSION_NUM < 120000 */ + if (arguments->backup_version <= 20021 || arguments->backup_version >= 20025) + crc = pgFileGetCRC32C(file_fullpath, false); + else + crc = pgFileGetCRC32(file_fullpath, false); +#endif /* PG_VERSION_NUM < 120000 */ + if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", @@ -720,8 +727,6 @@ validate_tablespace_map(pgBackup *backup, bool no_validate) pgFile **tablespace_map = NULL; pg_crc32 crc; parray *files = get_backup_filelist(backup, true); - bool use_crc32c = parse_program_version(backup->program_version) <= 20021 || - parse_program_version(backup->program_version) >= 20025; parray_qsort(files, pgFileCompareRelPathWithExternal); join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE); @@ -746,7 +751,16 @@ validate_tablespace_map(pgBackup *backup, bool no_validate) /* check tablespace map checksumms */ if (!no_validate) { - crc = pgFileGetCRC(map_path, use_crc32c, false); +#if PG_VERSION_NUM >= 120000 + Assert(parse_program_version(backup->program_version) >= 20025); + crc = pgFileGetCRC32C(map_path, false); +#else /* PG_VERSION_NUM < 120000 */ + if (parse_program_version(backup->program_version) <= 20021 + || parse_program_version(backup->program_version) >= 20025) + crc = pgFileGetCRC32C(map_path, false); + else + crc = pgFileGetCRC32(map_path, false); +#endif /* PG_VERSION_NUM < 120000 */ if ((*tablespace_map)->crc != crc) elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, "