From 64fac0c803e9d510a7d4d0bf1cd12e6b8062af51 Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Fri, 16 Oct 2020 09:12:46 -0700 Subject: [PATCH 1/9] Add no-alter-role option to pg_dumpall --- src/bin/pg_dump/pg_dumpall.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index c04f417cf053b..45b3be74489a5 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -82,6 +82,7 @@ static int no_role_passwords = 0; static int server_version; static int load_via_partition_root = 0; static int on_conflict_do_nothing = 0; +static int no_alter_role = 0; static char role_catalog[10]; #define PG_AUTHID "pg_authid" @@ -147,6 +148,7 @@ main(int argc, char *argv[]) {"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1}, {"on-conflict-do-nothing", no_argument, &on_conflict_do_nothing, 1}, {"rows-per-insert", required_argument, NULL, 7}, + {"no-alter-role", no_argument, &no_alter_role, 1}, {NULL, 0, NULL, 0} }; @@ -897,10 +899,17 @@ dumpRoles(PGconn *conn) * have failed to drop it. binary_upgrade cannot generate any errors, * so we assume the current role is already created. */ - if (!binary_upgrade || - strcmp(PQgetvalue(res, i, i_is_current_user), "f") == 0) - appendPQExpBuffer(buf, "CREATE ROLE %s;\n", fmtId(rolename)); - appendPQExpBuffer(buf, "ALTER ROLE %s WITH", fmtId(rolename)); + if (binary_upgrade) { + appendPQExpBuffer(buf, "ALTER ROLE %s WITH", fmtId(rolename)); + } else { + if (no_alter_role) { + appendPQExpBuffer(buf, "CREATE ROLE %s WITH", fmtId(rolename)); + } else { + if (strcmp(PQgetvalue(res, i, i_is_current_user), "f") == 0) + appendPQExpBuffer(buf, "CREATE ROLE %s;\n", fmtId(rolename)); + appendPQExpBuffer(buf, "ALTER ROLE %s WITH", fmtId(rolename)); + } + } if (strcmp(PQgetvalue(res, i, i_rolsuper), "t") == 0) appendPQExpBufferStr(buf, " SUPERUSER"); From 621a52093240b5b39f76315331d593842161c4b4 Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Fri, 16 Oct 2020 17:21:42 -0700 Subject: [PATCH 2/9] Add inih and merge-credentials-file options to pg_dumpall Due to limitations in CloudSQL we have the need to do effectively a pg_dumpall but with the ability to not perform ALTER ROLE and to merge passwords in to the statements. The alternative is to do this at runtime before passing to the restoration tool (psql), but it's nicer to have it at the source. This is pretty simple functionality. This does vendor in the inih library. No idea if problematic. --- src/bin/pg_dump/LICENSE-inih | 27 +++ src/bin/pg_dump/Makefile | 6 +- src/bin/pg_dump/ini.c | 298 +++++++++++++++++++++++++++++ src/bin/pg_dump/ini.h | 157 +++++++++++++++ src/bin/pg_dump/nls.mk | 4 +- src/bin/pg_dump/pg_dumpall.c | 69 ++++++- src/fe_utils/simple_list.c | 47 +++++ src/include/fe_utils/simple_list.h | 4 + src/tools/msvc/Mkvcbuild.pm | 1 + 9 files changed, 604 insertions(+), 9 deletions(-) create mode 100644 src/bin/pg_dump/LICENSE-inih create mode 100644 src/bin/pg_dump/ini.c create mode 100644 src/bin/pg_dump/ini.h diff --git a/src/bin/pg_dump/LICENSE-inih b/src/bin/pg_dump/LICENSE-inih new file mode 100644 index 0000000000000..6882deaa093a5 --- /dev/null +++ b/src/bin/pg_dump/LICENSE-inih @@ -0,0 +1,27 @@ + +The "inih" library is distributed under the New BSD license: + +Copyright (c) 2009, Ben Hoyt +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Ben Hoyt nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY BEN HOYT ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BEN HOYT BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index d3c1dce178e88..86000456f2bad 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -21,7 +21,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) OBJS= pg_backup_archiver.o pg_backup_db.o pg_backup_custom.o \ pg_backup_null.o pg_backup_tar.o pg_backup_directory.o \ - pg_backup_utils.o parallel.o compress_io.o dumputils.o $(WIN32RES) + pg_backup_utils.o parallel.o compress_io.o dumputils.o ini.o $(WIN32RES) all: pg_dump pg_restore pg_dumpall @@ -31,8 +31,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils $(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) -pg_dumpall: pg_dumpall.o dumputils.o | submake-libpq submake-libpgport submake-libpgfeutils - $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) +pg_dumpall: pg_dumpall.o dumputils.o ini.o | submake-libpq submake-libpgport submake-libpgfeutils + $(CC) $(CFLAGS) pg_dumpall.o dumputils.o ini.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) install: all installdirs $(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X) diff --git a/src/bin/pg_dump/ini.c b/src/bin/pg_dump/ini.c new file mode 100644 index 0000000000000..f2eec1fb21aab --- /dev/null +++ b/src/bin/pg_dump/ini.c @@ -0,0 +1,298 @@ +/* inih -- simple .INI file parser + +SPDX-License-Identifier: BSD-3-Clause + +Copyright (C) 2009-2020, Ben Hoyt + +inih is released under the New BSD license (see LICENSE.txt). Go to the project +home page for more info: + +https://github.com/benhoyt/inih + +*/ + +#if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif + +#include +#include +#include + +#include "ini.h" + +#if !INI_USE_STACK +#if INI_CUSTOM_ALLOCATOR +#include +void* ini_malloc(size_t size); +void ini_free(void* ptr); +void* ini_realloc(void* ptr, size_t size); +#else +#include +#define ini_malloc malloc +#define ini_free free +#define ini_realloc realloc +#endif +#endif + +#define MAX_SECTION 50 +#define MAX_NAME 50 + +/* Used by ini_parse_string() to keep track of string parsing state. */ +typedef struct { + const char* ptr; + size_t num_left; +} ini_parse_string_ctx; + +/* Strip whitespace chars off end of given string, in place. Return s. */ +static char* rstrip(char* s) +{ + char* p = s + strlen(s); + while (p > s && isspace((unsigned char)(*--p))) + *p = '\0'; + return s; +} + +/* Return pointer to first non-whitespace char in given string. */ +static char* lskip(const char* s) +{ + while (*s && isspace((unsigned char)(*s))) + s++; + return (char*)s; +} + +/* Return pointer to first char (of chars) or inline comment in given string, + or pointer to NUL at end of string if neither found. Inline comment must + be prefixed by a whitespace character to register as a comment. */ +static char* find_chars_or_comment(const char* s, const char* chars) +{ +#if INI_ALLOW_INLINE_COMMENTS + int was_space = 0; + while (*s && (!chars || !strchr(chars, *s)) && + !(was_space && strchr(INI_INLINE_COMMENT_PREFIXES, *s))) { + was_space = isspace((unsigned char)(*s)); + s++; + } +#else + while (*s && (!chars || !strchr(chars, *s))) { + s++; + } +#endif + return (char*)s; +} + +/* Similar to strncpy, but ensures dest (size bytes) is + NUL-terminated, and doesn't pad with NULs. */ +static char* strncpy0(char* dest, const char* src, size_t size) +{ + /* Could use strncpy internally, but it causes gcc warnings (see issue #91) */ + size_t i; + for (i = 0; i < size - 1 && src[i]; i++) + dest[i] = src[i]; + dest[i] = '\0'; + return dest; +} + +/* See documentation in header file. */ +int ini_parse_stream(ini_reader reader, void* stream, ini_handler handler, + void* user) +{ + /* Uses a fair bit of stack (use heap instead if you need to) */ +#if INI_USE_STACK + char line[INI_MAX_LINE]; + int max_line = INI_MAX_LINE; +#else + char* line; + size_t max_line = INI_INITIAL_ALLOC; +#endif +#if INI_ALLOW_REALLOC && !INI_USE_STACK + char* new_line; + size_t offset; +#endif + char section[MAX_SECTION] = ""; + char prev_name[MAX_NAME] = ""; + + char* start; + char* end; + char* name; + char* value; + int lineno = 0; + int error = 0; + +#if !INI_USE_STACK + line = (char*)ini_malloc(INI_INITIAL_ALLOC); + if (!line) { + return -2; + } +#endif + +#if INI_HANDLER_LINENO +#define HANDLER(u, s, n, v) handler(u, s, n, v, lineno) +#else +#define HANDLER(u, s, n, v) handler(u, s, n, v) +#endif + + /* Scan through stream line by line */ + while (reader(line, (int)max_line, stream) != NULL) { +#if INI_ALLOW_REALLOC && !INI_USE_STACK + offset = strlen(line); + while (offset == max_line - 1 && line[offset - 1] != '\n') { + max_line *= 2; + if (max_line > INI_MAX_LINE) + max_line = INI_MAX_LINE; + new_line = ini_realloc(line, max_line); + if (!new_line) { + ini_free(line); + return -2; + } + line = new_line; + if (reader(line + offset, (int)(max_line - offset), stream) == NULL) + break; + if (max_line >= INI_MAX_LINE) + break; + offset += strlen(line + offset); + } +#endif + + lineno++; + + start = line; +#if INI_ALLOW_BOM + if (lineno == 1 && (unsigned char)start[0] == 0xEF && + (unsigned char)start[1] == 0xBB && + (unsigned char)start[2] == 0xBF) { + start += 3; + } +#endif + start = lskip(rstrip(start)); + + if (strchr(INI_START_COMMENT_PREFIXES, *start)) { + /* Start-of-line comment */ + } +#if INI_ALLOW_MULTILINE + else if (*prev_name && *start && start > line) { + /* Non-blank line with leading whitespace, treat as continuation + of previous name's value (as per Python configparser). */ + if (!HANDLER(user, section, prev_name, start) && !error) + error = lineno; + } +#endif + else if (*start == '[') { + /* A "[section]" line */ + end = find_chars_or_comment(start + 1, "]"); + if (*end == ']') { + *end = '\0'; + strncpy0(section, start + 1, sizeof(section)); + *prev_name = '\0'; +#if INI_CALL_HANDLER_ON_NEW_SECTION + if (!HANDLER(user, section, NULL, NULL) && !error) + error = lineno; +#endif + } + else if (!error) { + /* No ']' found on section line */ + error = lineno; + } + } + else if (*start) { + /* Not a comment, must be a name[=:]value pair */ + end = find_chars_or_comment(start, "=:"); + if (*end == '=' || *end == ':') { + *end = '\0'; + name = rstrip(start); + value = end + 1; +#if INI_ALLOW_INLINE_COMMENTS + end = find_chars_or_comment(value, NULL); + if (*end) + *end = '\0'; +#endif + value = lskip(value); + rstrip(value); + + /* Valid name[=:]value pair found, call handler */ + strncpy0(prev_name, name, sizeof(prev_name)); + if (!HANDLER(user, section, name, value) && !error) + error = lineno; + } + else if (!error) { + /* No '=' or ':' found on name[=:]value line */ +#if INI_ALLOW_NO_VALUE + *end = '\0'; + name = rstrip(start); + if (!HANDLER(user, section, name, NULL) && !error) + error = lineno; +#else + error = lineno; +#endif + } + } + +#if INI_STOP_ON_FIRST_ERROR + if (error) + break; +#endif + } + +#if !INI_USE_STACK + ini_free(line); +#endif + + return error; +} + +/* See documentation in header file. */ +int ini_parse_file(FILE* file, ini_handler handler, void* user) +{ + return ini_parse_stream((ini_reader)fgets, file, handler, user); +} + +/* See documentation in header file. */ +int ini_parse(const char* filename, ini_handler handler, void* user) +{ + FILE* file; + int error; + + file = fopen(filename, "r"); + if (!file) + return -1; + error = ini_parse_file(file, handler, user); + fclose(file); + return error; +} + +/* An ini_reader function to read the next line from a string buffer. This + is the fgets() equivalent used by ini_parse_string(). */ +static char* ini_reader_string(char* str, int num, void* stream) { + ini_parse_string_ctx* ctx = (ini_parse_string_ctx*)stream; + const char* ctx_ptr = ctx->ptr; + size_t ctx_num_left = ctx->num_left; + char* strp = str; + char c; + + if (ctx_num_left == 0 || num < 2) + return NULL; + + while (num > 1 && ctx_num_left != 0) { + c = *ctx_ptr++; + ctx_num_left--; + *strp++ = c; + if (c == '\n') + break; + num--; + } + + *strp = '\0'; + ctx->ptr = ctx_ptr; + ctx->num_left = ctx_num_left; + return str; +} + +/* See documentation in header file. */ +int ini_parse_string(const char* string, ini_handler handler, void* user) { + ini_parse_string_ctx ctx; + + ctx.ptr = string; + ctx.num_left = strlen(string); + return ini_parse_stream((ini_reader)ini_reader_string, &ctx, handler, + user); +} \ No newline at end of file diff --git a/src/bin/pg_dump/ini.h b/src/bin/pg_dump/ini.h new file mode 100644 index 0000000000000..1a30f3c745ecc --- /dev/null +++ b/src/bin/pg_dump/ini.h @@ -0,0 +1,157 @@ +/* inih -- simple .INI file parser + +SPDX-License-Identifier: BSD-3-Clause + +Copyright (C) 2009-2020, Ben Hoyt + +inih is released under the New BSD license (see LICENSE.txt). Go to the project +home page for more info: + +https://github.com/benhoyt/inih + +*/ + +#ifndef INI_H +#define INI_H + +/* Make this header file easier to include in C++ code */ +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Nonzero if ini_handler callback should accept lineno parameter. */ +#ifndef INI_HANDLER_LINENO +#define INI_HANDLER_LINENO 0 +#endif + +/* Typedef for prototype of handler function. */ +#if INI_HANDLER_LINENO +typedef int (*ini_handler)(void* user, const char* section, + const char* name, const char* value, + int lineno); +#else +typedef int (*ini_handler)(void* user, const char* section, + const char* name, const char* value); +#endif + +/* Typedef for prototype of fgets-style reader function. */ +typedef char* (*ini_reader)(char* str, int num, void* stream); + +/* Parse given INI-style file. May have [section]s, name=value pairs + (whitespace stripped), and comments starting with ';' (semicolon). Section + is "" if name=value pair parsed before any section heading. name:value + pairs are also supported as a concession to Python's configparser. + + For each name=value pair parsed, call handler function with given user + pointer as well as section, name, and value (data only valid for duration + of handler call). Handler should return nonzero on success, zero on error. + + Returns 0 on success, line number of first error on parse error (doesn't + stop on first error), -1 on file open error, or -2 on memory allocation + error (only when INI_USE_STACK is zero). +*/ +int ini_parse(const char* filename, ini_handler handler, void* user); + +/* Same as ini_parse(), but takes a FILE* instead of filename. This doesn't + close the file when it's finished -- the caller must do that. */ +int ini_parse_file(FILE* file, ini_handler handler, void* user); + +/* Same as ini_parse(), but takes an ini_reader function pointer instead of + filename. Used for implementing custom or string-based I/O (see also + ini_parse_string). */ +int ini_parse_stream(ini_reader reader, void* stream, ini_handler handler, + void* user); + +/* Same as ini_parse(), but takes a zero-terminated string with the INI data +instead of a file. Useful for parsing INI data from a network socket or +already in memory. */ +int ini_parse_string(const char* string, ini_handler handler, void* user); + +/* Nonzero to allow multi-line value parsing, in the style of Python's + configparser. If allowed, ini_parse() will call the handler with the same + name for each subsequent line parsed. */ +#ifndef INI_ALLOW_MULTILINE +#define INI_ALLOW_MULTILINE 1 +#endif + +/* Nonzero to allow a UTF-8 BOM sequence (0xEF 0xBB 0xBF) at the start of + the file. See https://github.com/benhoyt/inih/issues/21 */ +#ifndef INI_ALLOW_BOM +#define INI_ALLOW_BOM 1 +#endif + +/* Chars that begin a start-of-line comment. Per Python configparser, allow + both ; and # comments at the start of a line by default. */ +#ifndef INI_START_COMMENT_PREFIXES +#define INI_START_COMMENT_PREFIXES ";#" +#endif + +/* Nonzero to allow inline comments (with valid inline comment characters + specified by INI_INLINE_COMMENT_PREFIXES). Set to 0 to turn off and match + Python 3.2+ configparser behaviour. */ +#ifndef INI_ALLOW_INLINE_COMMENTS +#define INI_ALLOW_INLINE_COMMENTS 1 +#endif +#ifndef INI_INLINE_COMMENT_PREFIXES +#define INI_INLINE_COMMENT_PREFIXES ";" +#endif + +/* Nonzero to use stack for line buffer, zero to use heap (malloc/free). */ +#ifndef INI_USE_STACK +#define INI_USE_STACK 1 +#endif + +/* Maximum line length for any line in INI file (stack or heap). Note that + this must be 3 more than the longest line (due to '\r', '\n', and '\0'). */ +#ifndef INI_MAX_LINE +#define INI_MAX_LINE 200 +#endif + +/* Nonzero to allow heap line buffer to grow via realloc(), zero for a + fixed-size buffer of INI_MAX_LINE bytes. Only applies if INI_USE_STACK is + zero. */ +#ifndef INI_ALLOW_REALLOC +#define INI_ALLOW_REALLOC 0 +#endif + +/* Initial size in bytes for heap line buffer. Only applies if INI_USE_STACK + is zero. */ +#ifndef INI_INITIAL_ALLOC +#define INI_INITIAL_ALLOC 200 +#endif + +/* Stop parsing on first error (default is to keep parsing). */ +#ifndef INI_STOP_ON_FIRST_ERROR +#define INI_STOP_ON_FIRST_ERROR 0 +#endif + +/* Nonzero to call the handler at the start of each new section (with + name and value NULL). Default is to only call the handler on + each name=value pair. */ +#ifndef INI_CALL_HANDLER_ON_NEW_SECTION +#define INI_CALL_HANDLER_ON_NEW_SECTION 0 +#endif + +/* Nonzero to allow a name without a value (no '=' or ':' on the line) and + call the handler with value NULL in this case. Default is to treat + no-value lines as an error. */ +#ifndef INI_ALLOW_NO_VALUE +#define INI_ALLOW_NO_VALUE 0 +#endif + +/* Nonzero to use custom ini_malloc, ini_free, and ini_realloc memory + allocation functions (INI_USE_STACK must also be 0). These functions must + have the same signatures as malloc/free/realloc and behave in a similar + way. ini_realloc is only needed if INI_ALLOW_REALLOC is set. */ +#ifndef INI_CUSTOM_ALLOCATOR +#define INI_CUSTOM_ALLOCATOR 0 +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* INI_H */ \ No newline at end of file diff --git a/src/bin/pg_dump/nls.mk b/src/bin/pg_dump/nls.mk index ec370209d0171..296f99cb29867 100644 --- a/src/bin/pg_dump/nls.mk +++ b/src/bin/pg_dump/nls.mk @@ -6,8 +6,8 @@ GETTEXT_FILES = $(FRONTEND_COMMON_GETTEXT_FILES) \ pg_backup_null.c pg_backup_tar.c \ pg_backup_directory.c dumputils.c compress_io.c \ pg_dump.c common.c pg_dump_sort.c \ - pg_restore.c pg_dumpall.c \ - parallel.c parallel.h pg_backup_utils.c pg_backup_utils.h \ + pg_restore.c pg_dumpall.c ini.c \ + parallel.c parallel.h pg_backup_utils.c pg_backup_utils.h ini.h \ ../../common/exec.c ../../common/fe_memutils.c \ ../../common/wait_error.c GETTEXT_TRIGGERS = $(FRONTEND_COMMON_GETTEXT_TRIGGERS) \ diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 45b3be74489a5..15d28748ce979 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -26,6 +26,7 @@ #include "common/logging.h" #include "fe_utils/connect.h" #include "fe_utils/string_utils.h" +#include "ini.h" /* version string we expect back from pg_dump */ #define PGDUMP_VERSIONSTR "pg_dump (PostgreSQL) " PG_VERSION "\n" @@ -96,6 +97,28 @@ static SimpleStringList database_exclude_names = {NULL, NULL}; #define exit_nicely(code) exit(code) +typedef struct +{ + SimpleStringList names; + SimpleStringList passwords; +} RoleCreds; +static RoleCreds role_creds = {{NULL, NULL}, {NULL, NULL}}; +static int merge_role_creds = 0; + +static int role_cred_handler(void* creds, const char* section, const char* name, const char* value) +{ + RoleCreds* pcreds = (RoleCreds*)creds; + + if (!simple_string_list_member(&(pcreds->names), name)) { + simple_string_list_append(&(pcreds->names), strdup(name)); + simple_string_list_append(&(pcreds->passwords), strdup(value)); + } else { + pg_log_error("ROLE %s specified more than once!", name); + return 0; + } + return 1; +} + int main(int argc, char *argv[]) { @@ -149,6 +172,7 @@ main(int argc, char *argv[]) {"on-conflict-do-nothing", no_argument, &on_conflict_do_nothing, 1}, {"rows-per-insert", required_argument, NULL, 7}, {"no-alter-role", no_argument, &no_alter_role, 1}, + {"merge-credentials-file", required_argument, NULL, 8}, {NULL, 0, NULL, 0} }; @@ -170,6 +194,7 @@ main(int argc, char *argv[]) int c, ret; int optindex; + char *merge_credentials_file = NULL; pg_logging_init(argv[0]); pg_logging_set_level(PG_LOG_WARNING); @@ -336,6 +361,11 @@ main(int argc, char *argv[]) appendPQExpBufferStr(pgdumpopts, " --rows-per-insert "); appendShellString(pgdumpopts, optarg); break; + + case 8: + merge_credentials_file = pg_strdup(optarg); + merge_role_creds = 1; + break; default: fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -393,6 +423,15 @@ main(int argc, char *argv[]) exit_nicely(1); } + if (merge_credentials_file != NULL) + { + if (ini_parse(merge_credentials_file, role_cred_handler, &role_creds) < 0) + { + pg_log_error("merge_credentials_file appears invalid."); + exit_nicely(1); + } + } + /* * If password values are not required in the dump, switch to using * pg_roles which is equally useful, just more likely to have unrestricted @@ -646,6 +685,9 @@ help(void) printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --inserts dump data as INSERT commands, rather than COPY\n")); printf(_(" --load-via-partition-root load partitions via the root table\n")); + printf(_(" --merge-credentials-file=FILENAME\n" + " merge passwords from file if not present\n")); + printf(_(" --no-alter-role do not alter role, only create\n")); printf(_(" --no-comments do not dump comments\n")); printf(_(" --no-publications do not dump publications\n")); printf(_(" --no-role-passwords do not dump passwords for roles\n")); @@ -899,12 +941,14 @@ dumpRoles(PGconn *conn) * have failed to drop it. binary_upgrade cannot generate any errors, * so we assume the current role is already created. */ - if (binary_upgrade) { + if (binary_upgrade) appendPQExpBuffer(buf, "ALTER ROLE %s WITH", fmtId(rolename)); - } else { - if (no_alter_role) { + else + { + if (no_alter_role) appendPQExpBuffer(buf, "CREATE ROLE %s WITH", fmtId(rolename)); - } else { + else + { if (strcmp(PQgetvalue(res, i, i_is_current_user), "f") == 0) appendPQExpBuffer(buf, "CREATE ROLE %s;\n", fmtId(rolename)); appendPQExpBuffer(buf, "ALTER ROLE %s WITH", fmtId(rolename)); @@ -956,6 +1000,23 @@ dumpRoles(PGconn *conn) appendPQExpBufferStr(buf, " PASSWORD "); appendStringLiteralConn(buf, PQgetvalue(res, i, i_rolpassword), conn); } + else if (merge_role_creds) + { + const int role_creds_index = simple_string_list_search(&(role_creds.names), rolename); + if (role_creds_index < 0) + pg_log_warning("No merge password specified for ROLE %s.", rolename); + else + { + const char *merge_password = simple_string_list_traverse(&(role_creds.passwords), role_creds_index); + if (merge_password == NULL) + pg_log_error("Merge password for ROLE %s is NULL. This should not happen.", rolename); + else + { + appendPQExpBufferStr(buf, " PASSWORD "); + appendStringLiteralConn(buf, merge_password, conn); + } + } + } if (!PQgetisnull(res, i, i_rolvaliduntil)) appendPQExpBuffer(buf, " VALID UNTIL '%s'", diff --git a/src/fe_utils/simple_list.c b/src/fe_utils/simple_list.c index 2232e8db73ef5..02582aea518f8 100644 --- a/src/fe_utils/simple_list.c +++ b/src/fe_utils/simple_list.c @@ -115,6 +115,53 @@ simple_string_list_not_touched(SimpleStringList *list) return NULL; } +/* + * Reset the touched state of the list entries. + */ +void +simple_string_list_reset_touched(SimpleStringList *list) +{ + SimpleStringListCell *cell; + + for (cell = list->head; cell; cell = cell->next) + { + cell->touched = false; + } +} + +/* + * Search for value in list and return its index. + */ +const int simple_string_list_search(SimpleStringList *list, const char *val) { + SimpleStringListCell *cell; + int index; + + for (cell = list->head, index = 0; cell; cell = cell->next, ++index) + { + if (strcmp(cell->val, val) == 0) + { + cell->touched = true; + return index; + } + } + return -1; +} + +/* + * Retrieve value at index. + */ +const char *simple_string_list_traverse(SimpleStringList *list, const int index) { + SimpleStringListCell *cell; + int curr_index; + + for (cell = list->head, curr_index = 0; cell && curr_index <= index; cell = cell->next, curr_index++) + { + if (curr_index == index) + return cell->val; + } + return NULL; +} + /* * Append a pointer to the list. * diff --git a/src/include/fe_utils/simple_list.h b/src/include/fe_utils/simple_list.h index 04dfa2cecfb4e..b22f8436bf3d3 100644 --- a/src/include/fe_utils/simple_list.h +++ b/src/include/fe_utils/simple_list.h @@ -62,6 +62,10 @@ extern void simple_string_list_append(SimpleStringList *list, const char *val); extern bool simple_string_list_member(SimpleStringList *list, const char *val); extern const char *simple_string_list_not_touched(SimpleStringList *list); +extern void simple_string_list_reset_touched(SimpleStringList *list); + +extern const int simple_string_list_search(SimpleStringList *list, const char *val); +extern const char *simple_string_list_traverse(SimpleStringList *list, const int iterations); extern void simple_ptr_list_append(SimplePtrList *list, void *val); diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index b13e57fa53193..667e3005e7c46 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -409,6 +409,7 @@ sub mkvcbuild $pgdumpall->AddIncludeDir('src/backend'); $pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c'); $pgdumpall->AddFile('src/bin/pg_dump/dumputils.c'); + $pgdumpall->AddFile('src/bin/pg_dump/ini.c'); $pgdumpall->AddLibrary('ws2_32.lib'); my $pgrestore = AddSimpleFrontend('pg_dump', 1); From d8554662fe6afe7e997067c716ae9b6aebea2a6a Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Mon, 19 Oct 2020 12:34:33 -0700 Subject: [PATCH 3/9] Add exclude-role flag to pg_dumpall --- src/bin/pg_dump/pg_dumpall.c | 97 +++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 15d28748ce979..0dd58b7b10667 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -95,6 +95,9 @@ static char *filename = NULL; static SimpleStringList database_exclude_patterns = {NULL, NULL}; static SimpleStringList database_exclude_names = {NULL, NULL}; +static SimpleStringList role_exclude_patterns = {NULL, NULL}; +static SimpleStringList role_exclude_names = {NULL, NULL}; + #define exit_nicely(code) exit(code) typedef struct @@ -173,6 +176,7 @@ main(int argc, char *argv[]) {"rows-per-insert", required_argument, NULL, 7}, {"no-alter-role", no_argument, &no_alter_role, 1}, {"merge-credentials-file", required_argument, NULL, 8}, + {"exclude-role", required_argument, NULL, 9}, {NULL, 0, NULL, 0} }; @@ -361,12 +365,16 @@ main(int argc, char *argv[]) appendPQExpBufferStr(pgdumpopts, " --rows-per-insert "); appendShellString(pgdumpopts, optarg); break; - + case 8: merge_credentials_file = pg_strdup(optarg); merge_role_creds = 1; break; + case 9: + simple_string_list_append(&role_exclude_patterns, optarg); + break; + default: fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit_nicely(1); @@ -515,6 +523,12 @@ main(int argc, char *argv[]) expand_dbname_patterns(conn, &database_exclude_patterns, &database_exclude_names); + /* + * Get a list of role names that match the exclude patterns + */ + expand_role_name_patterns(conn, &role_exclude_patterns, + &role_exclude_names); + /* * Open the output file if required, otherwise use stdout */ @@ -681,6 +695,7 @@ help(void) printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n")); printf(_(" --disable-triggers disable triggers during data-only restore\n")); printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n")); + printf(_(" --exclude-role=PATTERN exclude roles whose name matches PATTERN\n")); printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n")); printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --inserts dump data as INSERT commands, rather than COPY\n")); @@ -923,6 +938,15 @@ dumpRoles(PGconn *conn) continue; } + /* Skip any explicitly excluded roles */ + if (simple_string_list_member(&role_exclude_names, rolename)) + { + pg_log_info("excluding role \"%s\"", rolename); + continue; + } + + pg_log_info("dumping role \"%s\"", rolename); + resetPQExpBuffer(buf); if (binary_upgrade) @@ -1520,6 +1544,77 @@ expand_dbname_patterns(PGconn *conn, destroyPQExpBuffer(query); } +/* + * Find a list of role names that match the given patterns. + */ +static void +expand_role_name_patterns(PGconn *conn, + SimpleStringList *patterns, + SimpleStringList *names) +{ + PQExpBuffer query; + PGresult *res; + + if (patterns->head == NULL) + return; /* nothing to do */ + + query = createPQExpBuffer(); + + /* + * The loop below runs multiple SELECTs, which might sometimes result in + * duplicate entries in the name list, but we don't care, since all we're + * going to do is test membership of the list. + */ + + for (SimpleStringListCell *cell = patterns->head; cell; cell = cell->next) + { + if (server_version >= 90600) + appendPQExpBuffer(query, + "SELECT rolname " + "FROM %s " + "WHERE rolname !~ '^pg_'", + role_catalog); + else if (server_version >= 80100) + appendPQExpBuffer(query, + "SELECT rolname " + "FROM %s", + role_catalog); + else + appendPQExpBuffer(query, + "SELECT usename as rolname " + "FROM pg_shadow " + "UNION ALL " + "SELECT 0 as oid, groname as rolname, " + "false as rolsuper, " + "true as rolinherit, " + "false as rolcreaterole, " + "false as rolcreatedb, " + "false as rolcanlogin, " + "-1 as rolconnlimit, " + "null::text as rolpassword, " + "null::timestamptz as rolvaliduntil, " + "false as rolreplication, " + "false as rolbypassrls, " + "null as rolcomment, " + "false AS is_current_user " + "FROM pg_group " + "WHERE NOT EXISTS (SELECT 1 FROM pg_shadow " + "WHERE usename = groname)"); + processSQLNamePattern(conn, query, cell->val, false, + false, NULL, "rolname", NULL, NULL); + res = executeQuery(conn, query->data); + for (int i = 0; i < PQntuples(res); i++) + { + simple_string_list_append(names, PQgetvalue(res, i, 0)); + } + + PQclear(res); + resetPQExpBuffer(query); + } + + destroyPQExpBuffer(query); +} + /* * Dump contents of databases. */ From 7053070b38fa7b2a7c128dd7dba5e3707ea4df2f Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Mon, 19 Oct 2020 12:42:59 -0700 Subject: [PATCH 4/9] Declare expand_role_name_patterns function --- src/bin/pg_dump/pg_dumpall.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 0dd58b7b10667..0a526126efb14 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -56,6 +56,8 @@ static PGresult *executeQuery(PGconn *conn, const char *query); static void executeCommand(PGconn *conn, const char *query); static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns, SimpleStringList *names); +static void expand_role_name_patterns(PGconn *conn, SimpleStringList *patterns, + SimpleStringList *names); static char pg_dump_bin[MAXPGPATH]; static const char *progname; From 58736e17bf49d061cec1dafbf379919ba577519d Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Mon, 19 Oct 2020 14:29:49 -0700 Subject: [PATCH 5/9] WHERE pattern handling --- src/bin/pg_dump/pg_dumpall.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 0a526126efb14..358a8592af52b 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -1570,18 +1570,23 @@ expand_role_name_patterns(PGconn *conn, for (SimpleStringListCell *cell = patterns->head; cell; cell = cell->next) { + bool have_where = false; if (server_version >= 90600) + { appendPQExpBuffer(query, "SELECT rolname " "FROM %s " "WHERE rolname !~ '^pg_'", role_catalog); + have_where = true; + } else if (server_version >= 80100) appendPQExpBuffer(query, "SELECT rolname " - "FROM %s", + "FROM %s ", role_catalog); else + { appendPQExpBuffer(query, "SELECT usename as rolname " "FROM pg_shadow " @@ -1601,8 +1606,10 @@ expand_role_name_patterns(PGconn *conn, "false AS is_current_user " "FROM pg_group " "WHERE NOT EXISTS (SELECT 1 FROM pg_shadow " - "WHERE usename = groname)"); - processSQLNamePattern(conn, query, cell->val, false, + "WHERE usename = groname) "); + have_where = true; + } + processSQLNamePattern(conn, query, cell->val, have_where, false, NULL, "rolname", NULL, NULL); res = executeQuery(conn, query->data); for (int i = 0; i < PQntuples(res); i++) From d3857004c8994e22fda5d55c7d251124bfc08103 Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Mon, 19 Oct 2020 15:25:54 -0700 Subject: [PATCH 6/9] Option to remove granted by statements --- src/bin/pg_dump/pg_dumpall.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 358a8592af52b..2767dcad29f92 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -86,6 +86,7 @@ static int server_version; static int load_via_partition_root = 0; static int on_conflict_do_nothing = 0; static int no_alter_role = 0; +static int no_granted_by = 0; static char role_catalog[10]; #define PG_AUTHID "pg_authid" @@ -113,7 +114,6 @@ static int merge_role_creds = 0; static int role_cred_handler(void* creds, const char* section, const char* name, const char* value) { RoleCreds* pcreds = (RoleCreds*)creds; - if (!simple_string_list_member(&(pcreds->names), name)) { simple_string_list_append(&(pcreds->names), strdup(name)); simple_string_list_append(&(pcreds->passwords), strdup(value)); @@ -179,6 +179,7 @@ main(int argc, char *argv[]) {"no-alter-role", no_argument, &no_alter_role, 1}, {"merge-credentials-file", required_argument, NULL, 8}, {"exclude-role", required_argument, NULL, 9}, + {"no-granted-by", no_argument, &no_granted_by, 1}, {NULL, 0, NULL, 0} }; @@ -1030,12 +1031,14 @@ dumpRoles(PGconn *conn) { const int role_creds_index = simple_string_list_search(&(role_creds.names), rolename); if (role_creds_index < 0) - pg_log_warning("No merge password specified for ROLE %s.", rolename); + pg_log_warning("No merge PASSWORD specified for ROLE %s.", rolename); + if (strcmp(PQgetvalue(res, i, i_rolcanlogin), "t") == 0) + pg_log_error("ROLE %s has LOGIN and no PASSWORD for merge!", rolename) else { const char *merge_password = simple_string_list_traverse(&(role_creds.passwords), role_creds_index); if (merge_password == NULL) - pg_log_error("Merge password for ROLE %s is NULL. This should not happen.", rolename); + pg_log_error("Merge PASSWORD for ROLE %s is NULL. This should not happen.", rolename); else { appendPQExpBufferStr(buf, " PASSWORD "); @@ -1124,7 +1127,7 @@ dumpRoleMembership(PGconn *conn) * We don't track the grantor very carefully in the backend, so cope * with the possibility that it has been dropped. */ - if (!PQgetisnull(res, i, 3)) + if (!PQgetisnull(res, i, 3) && !no_granted_by) { char *grantor = PQgetvalue(res, i, 3); From 19aabed8efa07517eddeefc7c1827b0d17c17f5c Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Mon, 19 Oct 2020 15:34:09 -0700 Subject: [PATCH 7/9] Fix statements --- src/bin/pg_dump/pg_dumpall.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 2767dcad29f92..13a6fe3376080 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -1031,9 +1031,11 @@ dumpRoles(PGconn *conn) { const int role_creds_index = simple_string_list_search(&(role_creds.names), rolename); if (role_creds_index < 0) + { pg_log_warning("No merge PASSWORD specified for ROLE %s.", rolename); if (strcmp(PQgetvalue(res, i, i_rolcanlogin), "t") == 0) - pg_log_error("ROLE %s has LOGIN and no PASSWORD for merge!", rolename) + pg_log_error("ROLE %s has LOGIN and no PASSWORD for merge!", rolename); + } else { const char *merge_password = simple_string_list_traverse(&(role_creds.passwords), role_creds_index); From d658e64d5a70aa1d04d99ebffdb2880955679b6a Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Mon, 19 Oct 2020 15:53:55 -0700 Subject: [PATCH 8/9] Better error messaging --- src/bin/pg_dump/pg_dumpall.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 13a6fe3376080..f99a2600c61c0 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -1030,12 +1030,8 @@ dumpRoles(PGconn *conn) else if (merge_role_creds) { const int role_creds_index = simple_string_list_search(&(role_creds.names), rolename); - if (role_creds_index < 0) - { - pg_log_warning("No merge PASSWORD specified for ROLE %s.", rolename); - if (strcmp(PQgetvalue(res, i, i_rolcanlogin), "t") == 0) - pg_log_error("ROLE %s has LOGIN and no PASSWORD for merge!", rolename); - } + if (role_creds_index < 0 && strcmp(PQgetvalue(res, i, i_rolcanlogin), "t") == 0) + pg_log_warning("ROLE %s has LOGIN and no PASSWORD for merge!", rolename); else { const char *merge_password = simple_string_list_traverse(&(role_creds.passwords), role_creds_index); From 6a964b75eeef5374c0fd3a8121a2b86769078853 Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Tue, 20 Oct 2020 10:49:46 -0700 Subject: [PATCH 9/9] Add --no-granted-by doc --- src/bin/pg_dump/pg_dumpall.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index f99a2600c61c0..5c1115fb20a47 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -705,8 +705,9 @@ help(void) printf(_(" --load-via-partition-root load partitions via the root table\n")); printf(_(" --merge-credentials-file=FILENAME\n" " merge passwords from file if not present\n")); - printf(_(" --no-alter-role do not alter role, only create\n")); + printf(_(" --no-alter-role do not alter role, create with all attributes\n")); printf(_(" --no-comments do not dump comments\n")); + printf(_(" --no-granted-by do not dump GRANTED BY in ROLE statements\n")); printf(_(" --no-publications do not dump publications\n")); printf(_(" --no-role-passwords do not dump passwords for roles\n")); printf(_(" --no-security-labels do not dump security label assignments\n"));