From ce4b9193395852b270b0f6f3c8cdc6f5fb835233 Mon Sep 17 00:00:00 2001 From: Vladimir Rachkin Date: Tue, 25 Mar 2025 13:01:16 +0300 Subject: [PATCH 1/7] Remove unneccesary version checks from pg_dump Upstream has removed support for server versions older than 9.2.[1] The motivation for `error_unsupported_server_version` was to reduce diff footprint when upstream supported much older versions.[2] We still need to support GPDB5 (PostgreSQL 8.3), so remove version checks for PostgreSQL 8.2 and below. 1: https://github.com/postgres/postgres/commit/30e7c175b81d53c0f60f6ad12d1913a6d7d77008 2: https://github.com/bmdoil/gpdb/commit/b4c5a6186b1c0a1d9a3e2250a99546528de0c19e Authored-by: Brent Doil --- src/bin/pg_dump/common.c | 7 +- src/bin/pg_dump/dumputils.c | 14 - src/bin/pg_dump/pg_dump.c | 520 +++++------------------------------ src/bin/pg_dump/pg_dumpall.c | 118 +------- 4 files changed, 83 insertions(+), 576 deletions(-) diff --git a/src/bin/pg_dump/common.c b/src/bin/pg_dump/common.c index d3ff5cc69d8..822a7a1cf9c 100644 --- a/src/bin/pg_dump/common.c +++ b/src/bin/pg_dump/common.c @@ -186,11 +186,8 @@ getSchemaData(Archive *fout, int *numTablesPtr) oprinfo = getOperators(fout, &numOperators); oprinfoindex = buildIndexArray(oprinfo, numOperators, sizeof(OprInfo)); - if (testExtProtocolSupport(fout)) - { - pg_log_info("reading user-defined external protocols"); - getExtProtocols(fout, &numExtProtocols); - } + pg_log_info("reading user-defined external protocols"); + getExtProtocols(fout, &numExtProtocols); pg_log_info("reading user-defined access methods"); getAccessMethods(fout, &numAccessMethods); diff --git a/src/bin/pg_dump/dumputils.c b/src/bin/pg_dump/dumputils.c index 51ab650b46f..a481687cd5b 100644 --- a/src/bin/pg_dump/dumputils.c +++ b/src/bin/pg_dump/dumputils.c @@ -194,20 +194,6 @@ buildACLCommands(const char *name, const char *subname, const char *nspname, } } - /* - * We still need some hacking though to cover the case where new default - * public privileges are added in new versions: the REVOKE ALL will revoke - * them, leading to behavior different from what the old version had, - * which is generally not what's wanted. So add back default privs if the - * source database is too old to have had that particular priv. - */ - if (remoteVersion < 80200 && strcmp(type, "DATABASE") == 0) - { - /* database CONNECT priv didn't exist before 8.2 */ - appendPQExpBuffer(firstsql, "%sGRANT CONNECT ON %s %s TO PUBLIC;\n", - prefix, type, name); - } - /* Scan individual ACL items */ for (i = 0; i < naclitems; i++) { diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index a81ca91b3cb..36086d3b8d9 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -157,9 +157,6 @@ const char *EXT_PARTITION_NAME_POSTFIX = "_external_partition__"; /* flag indicating whether or not this GP database supports partitioning */ static bool gp_partitioning_available = false; -/* flag indicating whether or not this GP database supports column encoding */ -static bool gp_attribute_encoding_available = false; - static DumpId binary_upgrade_dumpid; /* override for standard extra_float_digits setting */ @@ -383,77 +380,17 @@ static void expand_oid_patterns(SimpleStringList *patterns, static bool is_returns_table_function(int nallargs, char **argmodes); static bool testGPbackend(Archive *fout); static bool testPartitioningSupport(Archive *fout); -static bool testAttributeEncodingSupport(Archive *fout); static char *nextToken(register char **stringp, register const char *delim); static void addDistributedBy(Archive *fout, PQExpBuffer q, const TableInfo *tbinfo, int actual_atts); static void addDistributedByOld(Archive *fout, PQExpBuffer q, const TableInfo *tbinfo, int actual_atts); static void addSchedule(Archive *fout, PQExpBuffer q, const TableInfo *tbinfo); -static bool isGPDB4300OrLater(Archive *fout); static bool isGPDB(Archive *fout); static bool isGPDB5000OrLater(Archive *fout); static bool isGPDB6000OrLater(Archive *fout); -static void error_unsupported_server_version(Archive *fout) pg_attribute_noreturn(); /* END MPP ADDITION */ -/* - * PostgreSQL's pg_dump supports very old server versions, but in GPDB, we - * only need to go back to 8.2-derived GPDB versions (4.something?). A lot of - * that code to deal with old versions has been removed. But in order to not - * change the formatting of the surrounding code, and to make it more clear - * when reading a diff against the corresponding PostgreSQL version of - * pg_dump, calls to this function has been left in place of the removed - * code. - * - * This function should never actually be used, because check that the server - * version is new enough at the beginning of pg_dump. This is just for - * documentation purposes, to show were upstream code has been removed, and - * to avoid those diffs or merge conflicts with upstream. - */ -static void -error_unsupported_server_version(Archive *fout) -{ - pg_log_warning("unexpected server version %d", fout->remoteVersion); - exit_nicely(1); -} - -/* - * If GPDB version is 4.3, pg_proc has prodataaccess column. - */ -static bool -isGPDB4300OrLater(Archive *fout) -{ - static int value = -1; /* -1 = not known yet, 0 = no, 1 = yes */ - - /* Query the server on first call, and cache the result */ - if (value == -1) - { - if (isGPbackend) - { - const char *query; - PGresult *res; - - query = "select attnum from pg_catalog.pg_attribute " - "where attrelid = 'pg_catalog.pg_proc'::regclass and " - "attname = 'prodataaccess'"; - - res = ExecuteSqlQuery(fout, query, PGRES_TUPLES_OK); - - if (PQntuples(res) == 1) - value = 1; - else - value = 0; - - PQclear(res); - } - else - value = 0; - } - - return (value == 1) ? true : false; -} - /* * Check if we are talking to GPDB */ @@ -1001,10 +938,10 @@ main(int argc, char **argv) /* - * We allow the server to be back to 8.0, and up to any minor release of + * We allow the server to be back to 8.3, and up to any minor release of * our own major version. (See also version check in pg_dumpall.c.) */ - fout->minRemoteVersion = 80200; /* we can handle back to 8.2 */ + fout->minRemoteVersion = 80300; /* we can handle back to 8.3 */ fout->maxRemoteVersion = (PG_VERSION_NUM / 100) * 100 + 99; fout->numWorkers = numWorkers; @@ -1054,22 +991,13 @@ main(int argc, char **argv) if (fout->isStandby) dopt.no_unlogged_table_data = true; - /* Select the appropriate subquery to convert user IDs to names */ - if (fout->remoteVersion >= 80100) - username_subquery = "SELECT rolname FROM pg_catalog.pg_roles WHERE oid ="; - else - error_unsupported_server_version(fout); + username_subquery = "SELECT rolname FROM pg_catalog.pg_roles WHERE oid ="; /* * Remember whether or not this GP database supports partitioning. */ gp_partitioning_available = testPartitioningSupport(fout); - /* - * Remember whether or not this GP database supports column encoding. - */ - gp_attribute_encoding_available = testAttributeEncodingSupport(fout); - /* check the version for the synchronized snapshots feature */ if (numWorkers > 1 && fout->remoteVersion < 90200 && !dopt.no_synchronized_snapshots) @@ -2421,7 +2349,6 @@ dumpTableData_copy(Archive *fout, const void *dcontext) */ if (tdinfo->filtercond || tbinfo->relkind == RELKIND_FOREIGN_TABLE) { - /* Note: this syntax is only supported in 8.2 and up */ appendPQExpBufferStr(q, "COPY (SELECT "); /* klugery to get rid of parens in column list */ if (strlen(column_list) > 2) @@ -3447,7 +3374,7 @@ dumpDatabase(Archive *fout) "WHERE datname = current_database()", username_subquery); } - else if (fout->remoteVersion >= 80200) + else { appendPQExpBuffer(dbQry, "SELECT tableoid, oid, datname, " "(%s datdba) AS dba, " @@ -3461,20 +3388,6 @@ dumpDatabase(Archive *fout) "WHERE datname = current_database()", username_subquery); } - else - { - error_unsupported_server_version(fout); - appendPQExpBuffer(dbQry, "SELECT tableoid, oid, datname, " - "(%s datdba) AS dba, " - "pg_encoding_to_char(encoding) AS encoding, " - "NULL AS datcollate, NULL AS datctype, datfrozenxid, 0 AS datminmxid, " - "datacl, '' as rdatacl, datistemplate, " - "-1 as datconnlimit, " - "(SELECT spcname FROM pg_tablespace t WHERE t.oid = dattablespace) AS tablespace " - "FROM pg_database " - "WHERE datname = current_database()", - username_subquery); - } res = ExecuteSqlQueryForSingleRow(fout, dbQry->data); @@ -3574,42 +3487,28 @@ dumpDatabase(Archive *fout) /* Compute correct tag for archive entry */ appendPQExpBuffer(labelq, "DATABASE %s", qdatname); - /* Dump DB comment if any */ - if (fout->remoteVersion >= 80200) - { - /* - * 8.2 and up keep comments on shared objects in a shared table, so we - * cannot use the dumpComment() code used for other database objects. - * Be careful that the ArchiveEntry parameters match that function. - */ - char *comment = PQgetvalue(res, 0, PQfnumber(res, "description")); + char *comment = PQgetvalue(res, 0, PQfnumber(res, "description")); - if (comment && *comment && !dopt->no_comments) - { - resetPQExpBuffer(dbQry); + if (comment && *comment && !dopt->no_comments) + { + resetPQExpBuffer(dbQry); - /* - * Generates warning when loaded into a differently-named - * database. - */ - appendPQExpBuffer(dbQry, "COMMENT ON DATABASE %s IS ", qdatname); - appendStringLiteralAH(dbQry, comment, fout); - appendPQExpBufferStr(dbQry, ";\n"); + /* + * Generates warning when loaded into a differently-named + * database. + */ + appendPQExpBuffer(dbQry, "COMMENT ON DATABASE %s IS ", qdatname); + appendStringLiteralAH(dbQry, comment, fout); + appendPQExpBufferStr(dbQry, ";\n"); - ArchiveEntry(fout, nilCatalogId, createDumpId(), - ARCHIVE_OPTS(.tag = labelq->data, - .owner = dba, - .description = "COMMENT", - .section = SECTION_NONE, - .createStmt = dbQry->data, - .deps = &dbDumpId, - .nDeps = 1)); - } - } - else - { - dumpComment(fout, "DATABASE", qdatname, NULL, dba, - dbCatId, 0, dbDumpId); + ArchiveEntry(fout, nilCatalogId, createDumpId(), + ARCHIVE_OPTS(.tag = labelq->data, + .owner = dba, + .description = "COMMENT", + .section = SECTION_NONE, + .createStmt = dbQry->data, + .deps = &dbDumpId, + .nDeps = 1)); } /* Dump DB security label, if enabled */ @@ -6090,7 +5989,7 @@ getTypes(Archive *fout, int *numTypes) "FROM pg_type", username_subquery); } - else if (fout->remoteVersion >= 80300) + else { appendPQExpBuffer(query, "SELECT tableoid, oid, typname, " "typnamespace, NULL AS typacl, NULL as rtypacl, " @@ -6105,21 +6004,6 @@ getTypes(Archive *fout, int *numTypes) "FROM pg_type", username_subquery); } - else - { - error_unsupported_server_version(fout); - appendPQExpBuffer(query, "SELECT tableoid, oid, typname, " - "typnamespace, NULL AS typacl, NULL as rtypacl, " - "NULL AS inittypacl, NULL AS initrtypacl, " - "(%s typowner) AS rolname, " - "typelem, typrelid, " - "CASE WHEN typrelid = 0 THEN ' '::\"char\" " - "ELSE (SELECT relkind FROM pg_class WHERE oid = typrelid) END AS typrelkind, " - "typtype, typisdefined, " - "typname[0] = '_' AND typelem != 0 AS isarray " - "FROM pg_type", - username_subquery); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -6266,15 +6150,6 @@ getTypeStorageOptions(Archive *fout, int *numTypes) int i_typoptions; int i_rolname; - if (gp_attribute_encoding_available == false) - { - numTypes = 0; - tstorageoptions = (TypeStorageOptions *) pg_malloc(0); - destroyPQExpBuffer(query); - return tstorageoptions; - } - - /* * The following statement used format_type to resolve an internal name to its equivalent sql name. * The format_type seems to do two things, it translates an internal type name (e.g. bpchar) into its @@ -6882,7 +6757,7 @@ getAggregates(Archive *fout, int *numAggs) destroyPQExpBuffer(initacl_subquery); destroyPQExpBuffer(initracl_subquery); } - else if (fout->remoteVersion >= 80200) + else { appendPQExpBuffer(query, "SELECT tableoid, oid, proname AS aggname, " "pronamespace AS aggnamespace, " @@ -6906,23 +6781,6 @@ getAggregates(Archive *fout, int *numAggs) "deptype = 'e')"); appendPQExpBufferChar(query, ')'); } - else - { - error_unsupported_server_version(fout); - appendPQExpBuffer(query, "SELECT tableoid, oid, proname AS aggname, " - "pronamespace AS aggnamespace, " - "CASE WHEN proargtypes[0] = 'pg_catalog.\"any\"'::pg_catalog.regtype THEN 0 ELSE 1 END AS pronargs, " - "proargtypes, " - "(%s proowner) AS rolname, " - "proacl AS aggacl, " - "NULL AS raggacl, " - "NULL AS initaggacl, NULL AS initraggacl " - "FROM pg_proc " - "WHERE proisagg " - "AND pronamespace != " - "(SELECT oid FROM pg_namespace WHERE nspname = 'pg_catalog')", - username_subquery); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -7972,7 +7830,7 @@ getTables(Archive *fout, int *numTables) RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, RELKIND_COMPOSITE_TYPE); } - else if (fout->remoteVersion >= 80200) + else { /* * Left join to pick up dependency info linking sequences to their @@ -8019,62 +7877,14 @@ getTables(Archive *fout, int *numTables) "LEFT JOIN pg_partition_rule pr ON c.oid = pr.parchildrelid " "LEFT JOIN pg_partition p ON pr.paroid = p.oid " "LEFT JOIN pg_partition pl ON (c.oid = pl.parrelid AND pl.parlevel = 0) " - "WHERE c.relkind in ('%c', '%c', '%c', '%c') %s " + "WHERE c.relkind in ('%c', '%c', '%c', '%c') " + "AND c.oid NOT IN (select p.parchildrelid from pg_partition_rule p " + "LEFT JOIN pg_exttable e on p.parchildrelid=e.reloid where e.reloid is null) " "AND c.relnamespace <> 3012 " /* BM_BITMAPINDEX_NAMESPACE in GPDB 5 and below */ "ORDER BY c.oid", username_subquery, RELKIND_SEQUENCE, RELKIND_RELATION, RELKIND_SEQUENCE, - RELKIND_VIEW, RELKIND_COMPOSITE_TYPE, - fout->remoteVersion >= 80209 ? - "AND c.oid NOT IN (select p.parchildrelid from pg_partition_rule p left " - "join pg_exttable e on p.parchildrelid=e.reloid where e.reloid is null)" : ""); - } - else - { - error_unsupported_server_version(fout); - /* - * Left join to pick up dependency info linking sequences to their - * owning column, if any - */ - appendPQExpBuffer(query, - "SELECT c.tableoid, c.oid, relname, " - "relacl, NULL as rrelacl, " - "NULL AS initrelacl, NULL AS initrrelacl, " - "relkind, relnamespace, " - "(%s relowner) AS rolname, " - "relchecks, (reltriggers <> 0) AS relhastriggers, " - "relhasindex, relhasrules, relhasoids, " - "'f'::bool AS relrowsecurity, " - "'f'::bool AS relforcerowsecurity, " - "0 AS relfrozenxid, 0 AS relminmxid," - "0 AS toid, " - "0 AS tfrozenxid, 0 AS tminmxid," - "'p' AS relpersistence, 't' as relispopulated, " - "'d' AS relreplident, relpages, " - "NULL AS amname, " - "NULL AS foreignserver, " - "NULL AS reloftype, " - "d.refobjid AS owning_tab, " - "d.refobjsubid AS owning_col, " - "(SELECT spcname FROM pg_tablespace t WHERE t.oid = c.reltablespace) AS reltablespace, " - "NULL AS reloptions, " - "NULL AS toast_reloptions, " - "NULL AS changed_acl, " - "NULL AS partkeydef, " - "false AS ispartition, " - "NULL AS partbound " - "FROM pg_class c " - "LEFT JOIN pg_depend d ON " - "(c.relkind = '%c' AND " - "d.classid = c.tableoid AND d.objid = c.oid AND " - "d.objsubid = 0 AND " - "d.refclassid = c.tableoid AND d.deptype = 'i') " - "WHERE relkind in ('%c', '%c', '%c', '%c') " - "ORDER BY c.oid", - username_subquery, - RELKIND_SEQUENCE, - RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, RELKIND_COMPOSITE_TYPE); } @@ -8618,7 +8428,7 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "ORDER BY indexname", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 80200) + else { appendPQExpBuffer(query, "SELECT t.tableoid, t.oid, " @@ -8652,40 +8462,6 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "ORDER BY indexname", tbinfo->dobj.catId.oid); } - else - { - error_unsupported_server_version(fout); - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "0 AS parentidx, " - "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " - "t.relnatts AS indnkeyatts, " - "t.relnatts AS indnatts, " - "i.indkey, i.indisclustered, " - "false AS indisreplident, " - "c.contype, c.conname, " - "c.condeferrable, c.condeferred, " - "c.tableoid AS contableoid, " - "c.oid AS conoid, " - "null AS condef, " - "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " - "null AS indreloptions, " - "'' AS indstatcols, " - "'' AS indstatvals " - "FROM pg_catalog.pg_index i " - "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " - "LEFT JOIN pg_catalog.pg_depend d " - "ON (d.classid = t.tableoid " - "AND d.objid = t.oid " - "AND d.deptype = 'i') " - "LEFT JOIN pg_catalog.pg_constraint c " - "ON (d.refclassid = c.tableoid " - "AND d.refobjid = c.oid) " - "WHERE i.indrelid = '%u'::pg_catalog.oid " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -9664,7 +9440,7 @@ getProcLangs(Archive *fout, int *numProcLangs) "ORDER BY oid", username_subquery); } - else if (fout->remoteVersion >= 80100) + else { /* Languages are owned by the bootstrap superuser, OID 10 */ appendPQExpBuffer(query, "SELECT tableoid, oid, " @@ -9678,21 +9454,6 @@ getProcLangs(Archive *fout, int *numProcLangs) "ORDER BY oid", username_subquery); } - else - { - error_unsupported_server_version(fout); - /* Languages are owned by the bootstrap superuser, sysid 1 */ - appendPQExpBuffer(query, "SELECT tableoid, oid, " - "lanname, lanpltrusted, lanplcallfoid, " - "0 AS laninline, lanvalidator, lanacl, " - "NULL AS rlanacl, " - "NULL AS initlanacl, NULL AS initrlanacl, " - "(%s '1') AS lanowner " - "FROM pg_language " - "WHERE lanispl " - "ORDER BY oid", - username_subquery); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -10115,15 +9876,6 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) ntups = PQntuples(res); i_attencoding = PQfnumber(res, "attencoding"); - /* - * attencoding is a Cloudberry specific column in the query, make sure - * it wasn't missed in a merge with PostgreSQL. - */ - if (gp_attribute_encoding_available && i_attencoding < 0) - { - pg_log_warning("attencoding column required in table attributes query"); - exit_nicely(1); - } tbinfo->numatts = ntups; tbinfo->attnames = (char **) pg_malloc(ntups * sizeof(char *)); @@ -10180,7 +9932,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->inhNotNull[j] = false; /* column storage attributes */ - if (gp_attribute_encoding_available && !PQgetisnull(res, j, i_attencoding)) + if (!PQgetisnull(res, j, i_attencoding)) tbinfo->attencoding[j] = pg_strdup(PQgetvalue(res, j, i_attencoding)); else tbinfo->attencoding[j] = NULL; @@ -13584,7 +13336,6 @@ dumpFunc(Archive *fout, const FuncInfo *finfo) char **allargtypes = NULL; char **argmodes = NULL; char **argnames = NULL; - bool isGE43 = isGPDB4300OrLater(fout); bool isGE50 = isGPDB5000OrLater(fout); char **configitems = NULL; int nconfigitems = 0; @@ -13652,7 +13403,7 @@ dumpFunc(Archive *fout, const FuncInfo *finfo) "%s\n" "'a' as proexeclocation,\n" "(SELECT procallback FROM pg_catalog.pg_proc_callback WHERE profnoid::pg_catalog.oid = p.oid) as callbackfunc,\n", - (isGE43 ? "prodataaccess," : "")); + "prodataaccess,"); } if (fout->remoteVersion >= 80400) @@ -14448,38 +14199,19 @@ dumpOpr(Archive *fout, const OprInfo *oprinfo) oprid = createPQExpBuffer(); details = createPQExpBuffer(); - if (fout->remoteVersion >= 80300) - { - appendPQExpBuffer(query, "SELECT oprkind, " - "oprcode::pg_catalog.regprocedure, " - "oprleft::pg_catalog.regtype, " - "oprright::pg_catalog.regtype, " - "oprcom, " - "oprnegate, " - "oprrest::pg_catalog.regprocedure, " - "oprjoin::pg_catalog.regprocedure, " - "oprcanmerge, oprcanhash " - "FROM pg_catalog.pg_operator " - "WHERE oid = '%u'::pg_catalog.oid", - oprinfo->dobj.catId.oid); - } - else - { - error_unsupported_server_version(fout); - appendPQExpBuffer(query, "SELECT oprkind, " - "oprcode::pg_catalog.regprocedure, " - "oprleft::pg_catalog.regtype, " - "oprright::pg_catalog.regtype, " - "oprcom, " - "oprnegate, " - "oprrest::pg_catalog.regprocedure, " - "oprjoin::pg_catalog.regprocedure, " - "(oprlsortop != 0) AS oprcanmerge, " - "oprcanhash " - "FROM pg_catalog.pg_operator " - "WHERE oid = '%u'::pg_catalog.oid", - oprinfo->dobj.catId.oid); - } + + appendPQExpBuffer(query, "SELECT oprkind, " + "oprcode::pg_catalog.regprocedure, " + "oprleft::pg_catalog.regtype, " + "oprright::pg_catalog.regtype, " + "oprcom, " + "oprnegate, " + "oprrest::pg_catalog.regprocedure, " + "oprjoin::pg_catalog.regprocedure, " + "oprcanmerge, oprcanhash " + "FROM pg_catalog.pg_operator " + "WHERE oid = '%u'::pg_catalog.oid", + oprinfo->dobj.catId.oid); res = ExecuteSqlQueryForSingleRow(fout, query->data); @@ -15774,13 +15506,6 @@ dumpAgg(Archive *fout, const AggInfo *agginfo) if (fout->remoteVersion >= 80100) appendPQExpBufferStr(query, "aggsortop,\n"); - else - { - /* GPDB can't be here */ - error_unsupported_server_version(fout); - appendPQExpBufferStr(query, - "0 AS aggsortop,\n"); - } if (fout->remoteVersion >= 80400) appendPQExpBufferStr(query, @@ -17564,7 +17289,6 @@ dumpExternal(Archive *fout, const TableInfo *tbinfo, PQExpBuffer q, PQExpBuffer bool isweb = false; bool iswritable = false; char *options; - bool gpdb5OrLater = isGPDB5000OrLater(fout); bool gpdb6OrLater = isGPDB6000OrLater(fout); char *logerrors = NULL; char *on_clause; @@ -17598,7 +17322,7 @@ dumpExternal(Archive *fout, const TableInfo *tbinfo, PQExpBuffer q, PQExpBuffer "FROM pg_catalog.pg_exttable x, pg_catalog.pg_class c " "WHERE x.reloid = c.oid AND c.oid = '%u'::oid ", tbinfo->dobj.catId.oid); } - else if (gpdb5OrLater) + else { appendPQExpBuffer(query, "SELECT x.urilocation, x.execlocation, x.fmttype, x.fmtopts, x.command, " @@ -17615,62 +17339,6 @@ dumpExternal(Archive *fout, const TableInfo *tbinfo, PQExpBuffer q, PQExpBuffer "FROM pg_catalog.pg_exttable x, pg_catalog.pg_class c " "WHERE x.reloid = c.oid AND c.oid = '%u'::oid ", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 80214) - { - appendPQExpBuffer(query, - "SELECT x.location, " - "CASE WHEN x.command <> '' THEN x.location " - "ELSE '{ALL_SEGMENTS}' " - "END AS execlocation, " - "x.fmttype, x.fmtopts, x.command, " - "x.rejectlimit, x.rejectlimittype, " - "d.relname AS logerrors, " - "pg_catalog.pg_encoding_to_char(x.encoding), " - "x.writable, null AS options " - "FROM pg_catalog.pg_class c " - "JOIN pg_catalog.pg_exttable x ON ( c.oid = x.reloid ) " - "LEFT JOIN pg_catalog.pg_class d ON ( d.oid = x.fmterrtbl ) " - "LEFT JOIN pg_catalog.pg_namespace n ON ( n.oid = d.relnamespace ) " - "WHERE c.oid = '%u'::oid ", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 80205) - { - - appendPQExpBuffer(query, - "SELECT x.location, " - "CASE WHEN x.command <> '' THEN x.location " - "ELSE '{ALL_SEGMENTS}' " - "END AS execlocation, " - "x.fmttype, x.fmtopts, x.command, " - "x.rejectlimit, x.rejectlimittype, " - "d.relname AS logerrors, " - "pg_catalog.pg_encoding_to_char(x.encoding), " - "null as writable, null as options " - "FROM pg_catalog.pg_class c " - "JOIN pg_catalog.pg_exttable x ON ( c.oid = x.reloid ) " - "LEFT JOIN pg_catalog.pg_class d ON ( d.oid = x.fmterrtbl ) " - "LEFT JOIN pg_catalog.pg_namespace n ON ( n.oid = d.relnamespace ) " - "WHERE c.oid = '%u'::oid ", - tbinfo->dobj.catId.oid); - } - else - { - /* not SREH and encoding columns yet */ - appendPQExpBuffer(query, - "SELECT x.location, " - "CASE WHEN x.command <> '' THEN x.location " - "ELSE '{ALL_SEGMENTS}' " - "END AS execlocation, " - "x.fmttype, x.fmtopts, x.command, " - "-1 as rejectlimit, null as rejectlimittype," - "null as logerrors, " - "null as encoding, null as writable, " - "null as options " - "FROM pg_catalog.pg_exttable x, pg_catalog.pg_class c " - "WHERE x.reloid = c.oid AND c.oid = '%u'::oid", - tbinfo->dobj.catId.oid); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -17867,36 +17535,33 @@ dumpExternal(Archive *fout, const TableInfo *tbinfo, PQExpBuffer q, PQExpBuffer appendPQExpBuffer(q, "OPTIONS (\n %s\n )\n", options); } - if (fout->remoteVersion >= 80205) + /* add ENCODING clause */ + appendPQExpBuffer(q, "ENCODING '%s'", extencoding); + + /* add Single Row Error Handling clause (if any) */ + if (rejlim && strlen(rejlim) > 0) { - /* add ENCODING clause */ - appendPQExpBuffer(q, "ENCODING '%s'", extencoding); + appendPQExpBufferChar(q, '\n'); - /* add Single Row Error Handling clause (if any) */ - if (rejlim && strlen(rejlim) > 0) + /* + * Error tables were removed and replaced with file error + * logging. + */ + if (logerrors && strlen(logerrors) > 0) { - appendPQExpBufferChar(q, '\n'); - - /* - * Error tables were removed and replaced with file error - * logging. - */ - if (logerrors && strlen(logerrors) > 0) - { - appendPQExpBufferStr(q, "LOG ERRORS "); - if (logerrors[0] == 'p' && strlen(logerrors) == 1) - appendPQExpBufferStr(q, "PERSISTENTLY "); - } + appendPQExpBufferStr(q, "LOG ERRORS "); + if (logerrors[0] == 'p' && strlen(logerrors) == 1) + appendPQExpBufferStr(q, "PERSISTENTLY "); + } - /* reject limit */ - appendPQExpBuffer(q, "SEGMENT REJECT LIMIT %s", rejlim); + /* reject limit */ + appendPQExpBuffer(q, "SEGMENT REJECT LIMIT %s", rejlim); - /* reject limit type */ - if (rejlimtype[0] == 'r') - appendPQExpBufferStr(q, " ROWS"); - else - appendPQExpBufferStr(q, " PERCENT"); - } + /* reject limit type */ + if (rejlimtype[0] == 'r') + appendPQExpBufferStr(q, " ROWS"); + else + appendPQExpBufferStr(q, " PERCENT"); } /* DISTRIBUTED BY clause (if WRITABLE table) */ @@ -21442,53 +21107,6 @@ testPartitioningSupport(Archive *fout) return isSupported; } - - -/* - * testAttributeEncodingSupport - tests whether or not the current GP - * database includes support for column encoding. - */ -static bool -testAttributeEncodingSupport(Archive *fout) -{ - PQExpBuffer query; - PGresult *res; - bool isSupported; - - query = createPQExpBuffer(); - - appendPQExpBuffer(query, "SELECT 1 from pg_catalog.pg_class where relnamespace = 11 and relname = 'pg_attribute_encoding';"); - res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); - - isSupported = (PQntuples(res) == 1); - - PQclear(res); - destroyPQExpBuffer(query); - - return isSupported; -} - - -bool -testExtProtocolSupport(Archive *fout) -{ - PQExpBuffer query; - PGresult *res; - bool isSupported; - - query = createPQExpBuffer(); - - appendPQExpBuffer(query, "SELECT 1 FROM pg_class WHERE relname = 'pg_extprotocol' and relnamespace = 11;"); - res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); - - isSupported = (PQntuples(res) == 1); - - PQclear(res); - destroyPQExpBuffer(query); - - return isSupported; -} - /* * addSchedule * diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 54b857e2678..73310951b54 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -64,8 +64,6 @@ static void executeCommand(PGconn *conn, const char *query); static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns, SimpleStringList *names); -static void error_unsupported_server_version(PGconn *conn) pg_attribute_noreturn(); - static char pg_dump_bin[MAXPGPATH]; static const char *progname; static PQExpBuffer pgdumpopts; @@ -925,22 +923,15 @@ dumpResQueues(PGconn *conn) "3 as ord FROM pg_resqueue " "UNION " "SELECT oid, rsqname, 'ignorecostlimit' as resname, " - "%s as ressetting, " + "rsqignorecostlimit::text as ressetting, " "4 as ord FROM pg_resqueue " - "%s" - "order by rsqname, ord", - (server_version >= 80205 ? - "rsqignorecostlimit::text" - : "0 AS"), - (server_version >= 80214 ? - "UNION " - "SELECT rq.oid, rq.rsqname, rt.resname, rc.ressetting, " - "rt.restypid as ord FROM " - "pg_resqueue rq, pg_resourcetype rt, " - "pg_resqueuecapability rc WHERE " - "rq.oid=rc.resqueueid and rc.restypid = rt.restypid " - : "") - ); + "UNION " + "SELECT rq.oid, rq.rsqname, rt.resname, rc.ressetting, " + "rt.restypid as ord FROM " + "pg_resqueue rq, pg_resourcetype rt, " + "pg_resqueuecapability rc WHERE " + "rq.oid=rc.resqueueid and rc.restypid = rt.restypid " + "order by rsqname, ord"); res = executeQuery(conn, buf->data); @@ -1237,7 +1228,7 @@ dumpRoles(PGconn *conn) " %s %s %s %s" "FROM %s " "ORDER BY 2", role_catalog, resq_col, resgroup_col, extauth_col, hdfs_col, role_catalog); - else if (server_version >= 80200) + else printfPQExpBuffer(buf, "SELECT oid, rolname, rolsuper, rolinherit, " "rolcreaterole, rolcreatedb, rolcatupdate, " @@ -1249,51 +1240,6 @@ dumpRoles(PGconn *conn) " %s %s %s %s" "FROM %s " "ORDER BY 2", role_catalog, resq_col, resgroup_col, extauth_col, hdfs_col, role_catalog); - else if (server_version >= 80100) - printfPQExpBuffer(buf, - "SELECT oid, rolname, rolsuper, rolinherit, " - "rolcreaterole, rolcreatedb, " - "rolcanlogin, rolconnlimit, rolpassword, " - "rolvaliduntil, false as rolreplication, " - "false as rolbypassrls, " - "null as rolcomment, " - "rolname = current_user AS is_current_user " - "FROM %s " - "ORDER BY 2", role_catalog); - else - printfPQExpBuffer(buf, - "SELECT 0 as oid, usename as rolname, " - "usesuper as rolsuper, " - "true as rolinherit, " - "usesuper as rolcreaterole, " - "usecreatedb as rolcreatedb, " - "true as rolcanlogin, " - "-1 as rolconnlimit, " - "passwd as rolpassword, " - "valuntil as rolvaliduntil, " - "false as rolreplication, " - "false as rolbypassrls, " - "null as rolcomment, " - "usename = current_user AS is_current_user " - "FROM pg_shadow " - "UNION ALL " - "SELECT 0 as oid, groname as rolname, " - "false as rolsuper, " - "true as rolinherit, " - "false as rolcreaterole, " - "false as rolcreatedb, " - "false as rolcanlogin, " - "-1 as rolconnlimit, " - "null::text as rolpassword, " - "null::timestamptz as rolvaliduntil, " - "false as rolreplication, " - "false as rolbypassrls, " - "null as rolcomment, " - "false AS is_current_user " - "FROM pg_group " - "WHERE NOT EXISTS (SELECT 1 FROM pg_shadow " - " WHERE usename = groname) " - "ORDER BY 2"); res = executeQuery(conn, buf->data); @@ -1701,11 +1647,6 @@ dumpTablespaces(PGconn *conn) * Note that we do not support initial privileges (pg_init_privs) on * tablespaces, so this logic cannot make use of buildACLQueries(). */ - if (server_version < 80214) - { - /* Filespaces were introduced in GP 4.0 (server_version 8.2.14) */ - return; - } if (server_version >= 140000) { @@ -1788,7 +1729,7 @@ dumpTablespaces(PGconn *conn) "FROM pg_catalog.pg_tablespace " "WHERE spcname !~ '^pg_' " "ORDER BY 1"); - else if (server_version >= 80200) + else res = executeQuery(conn, "SELECT oid, spcname, " "pg_catalog.pg_get_userbyid(spcowner) AS spcowner, " "spclocation, spcacl, '' as rspcacl, null, " @@ -1796,17 +1737,6 @@ dumpTablespaces(PGconn *conn) "FROM pg_catalog.pg_tablespace " "WHERE spcname !~ '^pg_' " "ORDER BY 1"); - else - { - error_unsupported_server_version(conn); - res = executeQuery(conn, "SELECT oid, spcname, " - "pg_catalog.pg_get_userbyid(spcowner) AS spcowner, " - "spclocation, spcacl, '' as rspcacl, " - "null, null, null " - "FROM pg_catalog.pg_tablespace " - "WHERE spcname !~ '^pg_' " - "ORDER BY 1"); - } if (PQntuples(res) > 0) fprintf(OPF, "--\n-- Tablespaces\n--\n\n"); @@ -2722,11 +2652,11 @@ connectDatabase(const char *dbname, const char *connection_string, my_version = PG_VERSION_NUM; /* - * We allow the server to be back to 8.0, and up to any minor release of + * We allow the server to be back to 8.3, and up to any minor release of * our own major version. (See also version check in pg_dump.c.) */ if (my_version != server_version - && (server_version < 80200 || /* we can handle back to 8.2 */ + && (server_version < 80300 || /* we can handle back to 8.3 */ (server_version / 100) > (my_version / 100))) { pg_log_error("server version: %s; %s version: %s", @@ -2837,27 +2767,3 @@ dumpTimestamp(const char *msg) if (strftime(buf, sizeof(buf), PGDUMP_STRFTIME_FMT, localtime(&now)) != 0) fprintf(OPF, "-- %s %s\n\n", msg, buf); } - -/* - * This GPDB-specific function is copied (in spirit) from pg_dump.c. - * - * PostgreSQL's pg_dumpall supports very old server versions, but in GPDB, we - * only need to go back to 8.2-derived GPDB versions (4.something?). A lot of - * that code to deal with old versions has been removed. But in order to not - * change the formatting of the surrounding code, and to make it more clear - * when reading a diff against the corresponding PostgreSQL version of - * pg_dumpall, calls to this function has been left in place of the removed - * code. - * - * This function should never actually be used, because check that the server - * version is new enough at the beginning of pg_dumpall. This is just for - * documentation purposes, to show were upstream code has been removed, and - * to avoid those diffs or merge conflicts with upstream. - */ -static void -error_unsupported_server_version(PGconn *conn) -{ - fprintf(stderr, _("unexpected server version %d\n"), server_version); - PQfinish(conn); - exit(1); -} From c31e9bd29da3f56aed826efa3546901d516832c7 Mon Sep 17 00:00:00 2001 From: Brent Doil Date: Thu, 9 Jun 2022 18:10:41 -0400 Subject: [PATCH 2/7] pg_dump: Remove unused flags and related code Removes the flags: - pre-data-schema-only - post-data-schema-only These are undocumented and unused. Removing them cleans up the code and reduces the diff with upstream. They were implemented to support `gpcrondump` which is no longer supported. Authored-by: Brent Doil --- src/bin/pg_dump/pg_dump.c | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 36086d3b8d9..4f9f6f985f9 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -100,8 +100,6 @@ static bool dosync = true; /* Issue fsync() to make dump durable on disk. */ /* START MPP ADDITION */ bool dumpGpPolicy; bool isGPbackend; -int preDataSchemaOnly; /* int because getopt_long() */ -int postDataSchemaOnly; /* END MPP ADDITION */ @@ -554,8 +552,6 @@ main(int argc, char **argv) */ {"gp-syntax", no_argument, NULL, 1000}, {"no-gp-syntax", no_argument, NULL, 1001}, - {"pre-data-schema-only", no_argument, &preDataSchemaOnly, 1}, - {"post-data-schema-only", no_argument, &postDataSchemaOnly, 1}, {"function-oids", required_argument, NULL, 1002}, {"relation-oids", required_argument, NULL, 1003}, /* END MPP ADDITION */ @@ -572,7 +568,6 @@ main(int argc, char **argv) */ init_parallel_dump_utils(); - preDataSchemaOnly = postDataSchemaOnly = false; progname = get_progname(argv[0]); if (argc > 1) @@ -844,10 +839,6 @@ main(int argc, char **argv) if (dopt.binary_upgrade) dopt.sequence_data = 1; - /* --pre-data-schema-only or --post-data-schema-only implies --schema-only */ - if (preDataSchemaOnly || postDataSchemaOnly) - dopt.schemaOnly = true; - if (dopt.dataOnly && dopt.schemaOnly) { pg_log_error("options -s/--schema-only and -a/--data-only cannot be used together"); @@ -11355,45 +11346,36 @@ dumpDumpableObject(Archive *fout, const DumpableObject *dobj) switch (dobj->objType) { case DO_NAMESPACE: - if (!postDataSchemaOnly) dumpNamespace(fout, (const NamespaceInfo *) dobj); break; case DO_EXTENSION: dumpExtension(fout, (const ExtensionInfo *) dobj); break; case DO_TYPE: - if (!postDataSchemaOnly) dumpType(fout, (const TypeInfo *) dobj); break; case DO_TYPE_STORAGE_OPTIONS: - if (!postDataSchemaOnly) - dumpTypeStorageOptions(fout, (const TypeStorageOptions *) dobj); + dumpTypeStorageOptions(fout, (const TypeStorageOptions *) dobj); break; case DO_SHELL_TYPE: - if (!postDataSchemaOnly) dumpShellType(fout, (const ShellTypeInfo *) dobj); break; case DO_FUNC: - if (!postDataSchemaOnly) dumpFunc(fout, (const FuncInfo *) dobj); break; case DO_AGG: - if (!postDataSchemaOnly) dumpAgg(fout, (const AggInfo *) dobj); break; case DO_EXTPROTOCOL: - if (!postDataSchemaOnly) dumpExtProtocol(fout, (const ExtProtInfo *) dobj); break; case DO_OPERATOR: - if (!postDataSchemaOnly) dumpOpr(fout, (const OprInfo *) dobj); break; case DO_ACCESS_METHOD: dumpAccessMethod(fout, (const AccessMethodInfo *) dobj); break; case DO_OPCLASS: - if (!postDataSchemaOnly) dumpOpclass(fout, (const OpclassInfo *) dobj); break; case DO_OPFAMILY: @@ -11403,11 +11385,9 @@ dumpDumpableObject(Archive *fout, const DumpableObject *dobj) dumpCollation(fout, (const CollInfo *) dobj); break; case DO_CONVERSION: - if (!postDataSchemaOnly) dumpConversion(fout, (const ConvInfo *) dobj); break; case DO_TABLE: - if (!postDataSchemaOnly) dumpTable(fout, (const TableInfo *) dobj); break; case DO_TABLE_ATTACH: @@ -11415,11 +11395,9 @@ dumpDumpableObject(Archive *fout, const DumpableObject *dobj) dumpTableAttach(fout, (const TableAttachInfo *) dobj); break; case DO_ATTRDEF: - if (!postDataSchemaOnly) dumpAttrDef(fout, (const AttrDefInfo *) dobj); break; case DO_INDEX: - if (!preDataSchemaOnly) dumpIndex(fout, (const IndxInfo *) dobj); break; case DO_INDEX_ATTACH: @@ -11432,7 +11410,6 @@ dumpDumpableObject(Archive *fout, const DumpableObject *dobj) refreshMatViewData(fout, (const TableDataInfo *) dobj); break; case DO_RULE: - if (!preDataSchemaOnly) dumpRule(fout, (const RuleInfo *) dobj); break; case DO_TRIGGER: @@ -11442,30 +11419,24 @@ dumpDumpableObject(Archive *fout, const DumpableObject *dobj) dumpEventTrigger(fout, (const EventTriggerInfo *) dobj); break; case DO_CONSTRAINT: - if (!preDataSchemaOnly) dumpConstraint(fout, (const ConstraintInfo *) dobj); break; case DO_FK_CONSTRAINT: - if (!preDataSchemaOnly) dumpConstraint(fout, (const ConstraintInfo *) dobj); break; case DO_PROCLANG: - if (!postDataSchemaOnly) dumpProcLang(fout, (const ProcLangInfo *) dobj); break; case DO_CAST: - if (!postDataSchemaOnly) dumpCast(fout, (const CastInfo *) dobj); break; case DO_TRANSFORM: - if (!postDataSchemaOnly) dumpTransform(fout, (const TransformInfo *) dobj); break; case DO_SEQUENCE_SET: dumpSequenceData(fout, (const TableDataInfo *) dobj); break; case DO_TABLE_DATA: - if (!postDataSchemaOnly) dumpTableData(fout, (const TableDataInfo *) dobj); break; case DO_DUMMY_TYPE: From 980a7bd502b6f1e9dc7bf501a4c5fff9818e6670 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 22 Oct 2021 17:19:03 -0400 Subject: [PATCH 3/7] In pg_dump, use simplehash.h to look up dumpable objects by OID. Backported from PG15 92316a4582a5714d4e494aaf90360860e7fec37a Conflicts resolved: * Move GPDB specific numTypeStorageOptions to getSchemaData function * Refactor getTypeStorageOptions query to return tableoid and oid fields, which are required to correctly index the hash table Original commit message follows: Create a hash table that indexes dumpable objects by CatalogId (that is, catalog OID + object OID). Use this to replace the former catalogIdMap array, as well as various other single- catalog index arrays, and also the extension membership map. In principle this should be faster for databases with many objects, since lookups are now O(1) not O(log N). However, it seems that these lookups are pretty much negligible in context, so that no overall performance change can be measured. But having only one lookup data structure to maintain makes the code simpler and more flexible, so let's do it anyway. Discussion: https://postgr.es/m/2595220.1634855245@sss.pgh.pa.us --- src/bin/pg_dump/common.c | 516 +++++++++++++++--------------------- src/bin/pg_dump/pg_backup.h | 1 + src/bin/pg_dump/pg_dump.c | 44 ++- src/bin/pg_dump/pg_dump.h | 13 +- 4 files changed, 227 insertions(+), 347 deletions(-) diff --git a/src/bin/pg_dump/common.c b/src/bin/pg_dump/common.c index 822a7a1cf9c..85515df381a 100644 --- a/src/bin/pg_dump/common.c +++ b/src/bin/pg_dump/common.c @@ -18,6 +18,14 @@ #include #include "catalog/pg_class_d.h" +#include "catalog/pg_collation_d.h" +#include "catalog/pg_extension_d.h" +#include "catalog/pg_namespace_d.h" +#include "catalog/pg_operator_d.h" +#include "catalog/pg_proc_d.h" +#include "catalog/pg_publication_d.h" +#include "catalog/pg_type_d.h" +#include "common/hashfn.h" #include "fe_utils/string_utils.h" #include "pg_backup_archiver.h" #include "pg_backup_utils.h" @@ -31,58 +39,54 @@ static int allocedDumpIds = 0; static DumpId lastDumpId = 0; /* Note: 0 is InvalidDumpId */ /* - * Variables for mapping CatalogId to DumpableObject - */ -static bool catalogIdMapValid = false; -static DumpableObject **catalogIdMap = NULL; -static int numCatalogIds = 0; - -/* - * These variables are static to avoid the notational cruft of having to pass - * them into findTableByOid() and friends. For each of these arrays, we build - * a sorted-by-OID index array immediately after the objects are fetched, - * and then we use binary search in findTableByOid() and friends. (qsort'ing - * the object arrays themselves would be simpler, but it doesn't work because - * pg_dump.c may have already established pointers between items.) + * Infrastructure for mapping CatalogId to DumpableObject + * + * We use a hash table generated by simplehash.h. That infrastructure + * requires all the hash table entries to be the same size, and it also + * expects that it can move them around when resizing the table. So we + * cannot make the DumpableObjects be elements of the hash table directly; + * instead, the hash table elements contain pointers to DumpableObjects. + * + * It turns out to be convenient to also use this data structure to map + * CatalogIds to owning extensions, if any. Since extension membership + * data is read before creating most DumpableObjects, either one of dobj + * and ext could be NULL. */ -static DumpableObject **tblinfoindex; -static DumpableObject **typinfoindex; -static DumpableObject **funinfoindex; -static DumpableObject **oprinfoindex; -static DumpableObject **collinfoindex; -static DumpableObject **nspinfoindex; -static DumpableObject **extinfoindex; -static DumpableObject **pubinfoindex; -static DumpableObject **binaryupgradeinfoindex; - -static int numTables; -static int numTypes; -static int numFuncs; -static int numOperators; -static int numCollations; -static int numNamespaces; -static int numExtensions; -static int numPublications; -static int numTypeStorageOptions; - - -/* This is an array of object identities, not actual DumpableObjects */ -static ExtensionMemberId *extmembers; -static int numextmembers; +typedef struct _catalogIdMapEntry +{ + CatalogId catId; /* the indexed CatalogId */ + uint32 status; /* hash status */ + uint32 hashval; /* hash code for the CatalogId */ + DumpableObject *dobj; /* the associated DumpableObject, if any */ + ExtensionInfo *ext; /* owning extension, if any */ +} CatalogIdMapEntry; + +#define SH_PREFIX catalogid +#define SH_ELEMENT_TYPE CatalogIdMapEntry +#define SH_KEY_TYPE CatalogId +#define SH_KEY catId +#define SH_HASH_KEY(tb, key) hash_bytes((const unsigned char *) &(key), sizeof(CatalogId)) +#define SH_EQUAL(tb, a, b) ((a).oid == (b).oid && (a).tableoid == (b).tableoid) +#define SH_STORE_HASH +#define SH_GET_HASH(tb, a) (a)->hashval +#define SH_SCOPE static inline +#define SH_RAW_ALLOCATOR pg_malloc0 +#define SH_DECLARE +#define SH_DEFINE +#include "lib/simplehash.h" + +#define CATALOGIDHASH_INITIAL_SIZE 10000 + +static catalogid_hash *catalogIdHash = NULL; static void flagInhTables(Archive *fout, TableInfo *tbinfo, int numTables, InhInfo *inhinfo, int numInherits); static void flagInhIndexes(Archive *fout, TableInfo *tblinfo, int numTables); static void flagInhAttrs(DumpOptions *dopt, TableInfo *tblinfo, int numTables); -static DumpableObject **buildIndexArray(void *objArray, int numObjs, - Size objSize); -static int DOCatalogIdCompare(const void *p1, const void *p2); -static int ExtensionMemberIdCompare(const void *p1, const void *p2); static void findParentsByOid(TableInfo *self, InhInfo *inhinfo, int numInherits); static int strInArray(const char *pattern, char **arr, int arr_size); -static IndxInfo *findIndexByOid(Oid oid, DumpableObject **idxinfoindex, - int numIndexes); +static IndxInfo *findIndexByOid(Oid oid); /* @@ -93,14 +97,16 @@ TableInfo * getSchemaData(Archive *fout, int *numTablesPtr) { TableInfo *tblinfo; - TypeInfo *typinfo; - FuncInfo *funinfo; - OprInfo *oprinfo; - CollInfo *collinfo; - NamespaceInfo *nspinfo; ExtensionInfo *extinfo; - PublicationInfo *pubinfo; InhInfo *inhinfo; + int numTables; + int numTypes; + int numFuncs; + int numOperators; + int numCollations; + int numNamespaces; + int numExtensions; + int numPublications; int numAggregates; int numInherits; int numRules; @@ -119,6 +125,7 @@ getSchemaData(Archive *fout, int *numTablesPtr) int numForeignServers; int numDefaultACLs; int numEventTriggers; + int numTypeStorageOptions; /* GPDB specific variables */ int numExtProtocols; @@ -130,7 +137,6 @@ getSchemaData(Archive *fout, int *numTablesPtr) pg_log_info("identifying required binary upgrade calls"); binfo = getBinaryUpgradeObjects(); - binaryupgradeinfoindex = buildIndexArray(binfo, 1, sizeof(BinaryUpgradeInfo)); } /* @@ -140,14 +146,12 @@ getSchemaData(Archive *fout, int *numTablesPtr) */ pg_log_info("reading extensions"); extinfo = getExtensions(fout, &numExtensions); - extinfoindex = buildIndexArray(extinfo, numExtensions, sizeof(ExtensionInfo)); pg_log_info("identifying extension members"); getExtensionMembership(fout, extinfo, numExtensions); pg_log_info("reading schemas"); - nspinfo = getNamespaces(fout, &numNamespaces); - nspinfoindex = buildIndexArray(nspinfo, numNamespaces, sizeof(NamespaceInfo)); + (void) getNamespaces(fout, &numNamespaces); /* * getTables should be done as soon as possible, so as to minimize the @@ -157,19 +161,15 @@ getSchemaData(Archive *fout, int *numTablesPtr) */ pg_log_info("reading user-defined tables"); tblinfo = getTables(fout, &numTables); - tblinfoindex = buildIndexArray(tblinfo, numTables, sizeof(TableInfo)); - /* Do this after we've built tblinfoindex */ getOwnedSeqs(fout, tblinfo, numTables); pg_log_info("reading user-defined functions"); - funinfo = getFuncs(fout, &numFuncs); - funinfoindex = buildIndexArray(funinfo, numFuncs, sizeof(FuncInfo)); + (void) getFuncs(fout, &numFuncs); /* this must be after getTables and getFuncs */ pg_log_info("reading user-defined types"); - typinfo = getTypes(fout, &numTypes); - typinfoindex = buildIndexArray(typinfo, numTypes, sizeof(TypeInfo)); + (void) getTypes(fout, &numTypes); /* this must be after getFuncs */ pg_log_info("reading type storage options"); @@ -183,8 +183,7 @@ getSchemaData(Archive *fout, int *numTablesPtr) getAggregates(fout, &numAggregates); pg_log_info("reading user-defined operators"); - oprinfo = getOperators(fout, &numOperators); - oprinfoindex = buildIndexArray(oprinfo, numOperators, sizeof(OprInfo)); + (void) getOperators(fout, &numOperators); pg_log_info("reading user-defined external protocols"); getExtProtocols(fout, &numExtProtocols); @@ -220,8 +219,7 @@ getSchemaData(Archive *fout, int *numTablesPtr) getDefaultACLs(fout, &numDefaultACLs); pg_log_info("reading user-defined collations"); - collinfo = getCollations(fout, &numCollations); - collinfoindex = buildIndexArray(collinfo, numCollations, sizeof(CollInfo)); + (void) getCollations(fout, &numCollations); pg_log_info("reading user-defined conversions"); getConversions(fout, &numConversions); @@ -274,9 +272,7 @@ getSchemaData(Archive *fout, int *numTablesPtr) getPolicies(fout, tblinfo, numTables); pg_log_info("reading publications"); - pubinfo = getPublications(fout, &numPublications); - pubinfoindex = buildIndexArray(pubinfo, numPublications, - sizeof(PublicationInfo)); + (void) getPublications(fout, &numPublications); pg_log_info("reading publication membership"); getPublicationTables(fout, tblinfo, numTables); @@ -407,34 +403,15 @@ flagInhIndexes(Archive *fout, TableInfo tblinfo[], int numTables) int i, j, k; - DumpableObject ***parentIndexArray; - - parentIndexArray = (DumpableObject ***) - pg_malloc0(getMaxDumpId() * sizeof(DumpableObject **)); for (i = 0; i < numTables; i++) { - TableInfo *parenttbl; IndexAttachInfo *attachinfo; if (!tblinfo[i].ispartition || tblinfo[i].numParents == 0) continue; Assert(tblinfo[i].numParents == 1); - parenttbl = tblinfo[i].parents[0]; - - /* - * We need access to each parent table's index list, but there is no - * index to cover them outside of this function. To avoid having to - * sort every parent table's indexes each time we come across each of - * its partitions, create an indexed array for each parent the first - * time it is required. - */ - if (parentIndexArray[parenttbl->dobj.dumpId] == NULL) - parentIndexArray[parenttbl->dobj.dumpId] = - buildIndexArray(parenttbl->indexes, - parenttbl->numIndexes, - sizeof(IndxInfo)); attachinfo = (IndexAttachInfo *) pg_malloc0(tblinfo[i].numIndexes * sizeof(IndexAttachInfo)); @@ -446,9 +423,7 @@ flagInhIndexes(Archive *fout, TableInfo tblinfo[], int numTables) if (index->parentidx == 0) continue; - parentidx = findIndexByOid(index->parentidx, - parentIndexArray[parenttbl->dobj.dumpId], - parenttbl->numIndexes); + parentidx = findIndexByOid(index->parentidx); if (parentidx == NULL) continue; @@ -489,11 +464,6 @@ flagInhIndexes(Archive *fout, TableInfo tblinfo[], int numTables) k++; } } - - for (i = 0; i < numTables; i++) - if (parentIndexArray[i]) - pg_free(parentIndexArray[i]); - pg_free(parentIndexArray); } /* flagInhAttrs - @@ -636,7 +606,7 @@ flagInhAttrs(DumpOptions *dopt, TableInfo *tblinfo, int numTables) /* * AssignDumpId * Given a newly-created dumpable object, assign a dump ID, - * and enter the object into the lookup table. + * and enter the object into the lookup tables. * * The caller is expected to have filled in objType and catId, * but not any of the other standard fields of a DumpableObject. @@ -655,6 +625,7 @@ AssignDumpId(DumpableObject *dobj) dobj->nDeps = 0; dobj->allocDeps = 0; + /* Add object to dumpIdMap[], enlarging that array if need be */ while (dobj->dumpId >= allocedDumpIds) { int newAlloc; @@ -677,8 +648,25 @@ AssignDumpId(DumpableObject *dobj) } dumpIdMap[dobj->dumpId] = dobj; - /* mark catalogIdMap invalid, but don't rebuild it yet */ - catalogIdMapValid = false; + /* If it has a valid CatalogId, enter it into the hash table */ + if (OidIsValid(dobj->catId.tableoid)) + { + CatalogIdMapEntry *entry; + bool found; + + /* Initialize CatalogId hash table if not done yet */ + if (catalogIdHash == NULL) + catalogIdHash = catalogid_create(CATALOGIDHASH_INITIAL_SIZE, NULL); + + entry = catalogid_insert(catalogIdHash, dobj->catId, &found); + if (!found) + { + entry->dobj = NULL; + entry->ext = NULL; + } + Assert(entry->dobj == NULL); + entry->dobj = dobj; + } } /* @@ -719,140 +707,19 @@ findObjectByDumpId(DumpId dumpId) * Find a DumpableObject by catalog ID * * Returns NULL for unknown ID - * - * We use binary search in a sorted list that is built on first call. - * If AssignDumpId() and findObjectByCatalogId() calls were freely intermixed, - * the code would work, but possibly be very slow. In the current usage - * pattern that does not happen, indeed we build the list at most twice. */ DumpableObject * findObjectByCatalogId(CatalogId catalogId) { - DumpableObject **low; - DumpableObject **high; + CatalogIdMapEntry *entry; - if (!catalogIdMapValid) - { - if (catalogIdMap) - free(catalogIdMap); - getDumpableObjects(&catalogIdMap, &numCatalogIds); - if (numCatalogIds > 1) - qsort((void *) catalogIdMap, numCatalogIds, - sizeof(DumpableObject *), DOCatalogIdCompare); - catalogIdMapValid = true; - } + if (catalogIdHash == NULL) + return NULL; /* no objects exist yet */ - /* - * We could use bsearch() here, but the notational cruft of calling - * bsearch is nearly as bad as doing it ourselves; and the generalized - * bsearch function is noticeably slower as well. - */ - if (numCatalogIds <= 0) + entry = catalogid_lookup(catalogIdHash, catalogId); + if (entry == NULL) return NULL; - low = catalogIdMap; - high = catalogIdMap + (numCatalogIds - 1); - while (low <= high) - { - DumpableObject **middle; - int difference; - - middle = low + (high - low) / 2; - /* comparison must match DOCatalogIdCompare, below */ - difference = oidcmp((*middle)->catId.oid, catalogId.oid); - if (difference == 0) - difference = oidcmp((*middle)->catId.tableoid, catalogId.tableoid); - if (difference == 0) - return *middle; - else if (difference < 0) - low = middle + 1; - else - high = middle - 1; - } - return NULL; -} - -/* - * Find a DumpableObject by OID, in a pre-sorted array of one type of object - * - * Returns NULL for unknown OID - */ -static DumpableObject * -findObjectByOid(Oid oid, DumpableObject **indexArray, int numObjs) -{ - DumpableObject **low; - DumpableObject **high; - - /* - * This is the same as findObjectByCatalogId except we assume we need not - * look at table OID because the objects are all the same type. - * - * We could use bsearch() here, but the notational cruft of calling - * bsearch is nearly as bad as doing it ourselves; and the generalized - * bsearch function is noticeably slower as well. - */ - if (numObjs <= 0) - return NULL; - low = indexArray; - high = indexArray + (numObjs - 1); - while (low <= high) - { - DumpableObject **middle; - int difference; - - middle = low + (high - low) / 2; - difference = oidcmp((*middle)->catId.oid, oid); - if (difference == 0) - return *middle; - else if (difference < 0) - low = middle + 1; - else - high = middle - 1; - } - return NULL; -} - -/* - * Build an index array of DumpableObject pointers, sorted by OID - */ -static DumpableObject ** -buildIndexArray(void *objArray, int numObjs, Size objSize) -{ - DumpableObject **ptrs; - int i; - - if (numObjs <= 0) - return NULL; - - ptrs = (DumpableObject **) pg_malloc(numObjs * sizeof(DumpableObject *)); - for (i = 0; i < numObjs; i++) - ptrs[i] = (DumpableObject *) ((char *) objArray + i * objSize); - - /* We can use DOCatalogIdCompare to sort since its first key is OID */ - if (numObjs > 1) - qsort((void *) ptrs, numObjs, sizeof(DumpableObject *), - DOCatalogIdCompare); - - return ptrs; -} - -/* - * qsort comparator for pointers to DumpableObjects - */ -static int -DOCatalogIdCompare(const void *p1, const void *p2) -{ - const DumpableObject *obj1 = *(DumpableObject *const *) p1; - const DumpableObject *obj2 = *(DumpableObject *const *) p2; - int cmpval; - - /* - * Compare OID first since it's usually unique, whereas there will only be - * a few distinct values of tableoid. - */ - cmpval = oidcmp(obj1->catId.oid, obj2->catId.oid); - if (cmpval == 0) - cmpval = oidcmp(obj1->catId.tableoid, obj2->catId.tableoid); - return cmpval; + return entry->dobj; } /* @@ -926,119 +793,191 @@ removeObjectDependency(DumpableObject *dobj, DumpId refId) /* * findTableByOid - * finds the entry (in tblinfo) of the table with the given oid + * finds the DumpableObject for the table with the given oid * returns NULL if not found */ TableInfo * findTableByOid(Oid oid) { - return (TableInfo *) findObjectByOid(oid, tblinfoindex, numTables); + CatalogId catId; + DumpableObject *dobj; + + catId.tableoid = RelationRelationId; + catId.oid = oid; + dobj = findObjectByCatalogId(catId); + Assert(dobj == NULL || dobj->objType == DO_TABLE); + return (TableInfo *) dobj; +} + +/* + * findIndexByOid + * finds the DumpableObject for the index with the given oid + * returns NULL if not found + */ +static IndxInfo * +findIndexByOid(Oid oid) +{ + CatalogId catId; + DumpableObject *dobj; + + catId.tableoid = RelationRelationId; + catId.oid = oid; + dobj = findObjectByCatalogId(catId); + Assert(dobj == NULL || dobj->objType == DO_INDEX); + return (IndxInfo *) dobj; } /* * findTypeByOid - * finds the entry (in typinfo) of the type with the given oid + * finds the DumpableObject for the type with the given oid * returns NULL if not found */ TypeInfo * findTypeByOid(Oid oid) { - return (TypeInfo *) findObjectByOid(oid, typinfoindex, numTypes); + CatalogId catId; + DumpableObject *dobj; + + catId.tableoid = TypeRelationId; + catId.oid = oid; + dobj = findObjectByCatalogId(catId); + Assert(dobj == NULL || + dobj->objType == DO_TYPE || dobj->objType == DO_DUMMY_TYPE); + return (TypeInfo *) dobj; } /* * findFuncByOid - * finds the entry (in funinfo) of the function with the given oid + * finds the DumpableObject for the function with the given oid * returns NULL if not found */ FuncInfo * findFuncByOid(Oid oid) { - return (FuncInfo *) findObjectByOid(oid, funinfoindex, numFuncs); + CatalogId catId; + DumpableObject *dobj; + + catId.tableoid = ProcedureRelationId; + catId.oid = oid; + dobj = findObjectByCatalogId(catId); + Assert(dobj == NULL || dobj->objType == DO_FUNC); + return (FuncInfo *) dobj; } /* * findOprByOid - * finds the entry (in oprinfo) of the operator with the given oid + * finds the DumpableObject for the operator with the given oid * returns NULL if not found */ OprInfo * findOprByOid(Oid oid) { - return (OprInfo *) findObjectByOid(oid, oprinfoindex, numOperators); + CatalogId catId; + DumpableObject *dobj; + + catId.tableoid = OperatorRelationId; + catId.oid = oid; + dobj = findObjectByCatalogId(catId); + Assert(dobj == NULL || dobj->objType == DO_OPERATOR); + return (OprInfo *) dobj; } /* * findCollationByOid - * finds the entry (in collinfo) of the collation with the given oid + * finds the DumpableObject for the collation with the given oid * returns NULL if not found */ CollInfo * findCollationByOid(Oid oid) { - return (CollInfo *) findObjectByOid(oid, collinfoindex, numCollations); + CatalogId catId; + DumpableObject *dobj; + + catId.tableoid = CollationRelationId; + catId.oid = oid; + dobj = findObjectByCatalogId(catId); + Assert(dobj == NULL || dobj->objType == DO_COLLATION); + return (CollInfo *) dobj; } /* * findNamespaceByOid - * finds the entry (in nspinfo) of the namespace with the given oid + * finds the DumpableObject for the namespace with the given oid * returns NULL if not found */ NamespaceInfo * findNamespaceByOid(Oid oid) { - return (NamespaceInfo *) findObjectByOid(oid, nspinfoindex, numNamespaces); + CatalogId catId; + DumpableObject *dobj; + + catId.tableoid = NamespaceRelationId; + catId.oid = oid; + dobj = findObjectByCatalogId(catId); + Assert(dobj == NULL || dobj->objType == DO_NAMESPACE); + return (NamespaceInfo *) dobj; } /* * findExtensionByOid - * finds the entry (in extinfo) of the extension with the given oid + * finds the DumpableObject for the extension with the given oid * returns NULL if not found */ ExtensionInfo * findExtensionByOid(Oid oid) { - return (ExtensionInfo *) findObjectByOid(oid, extinfoindex, numExtensions); + CatalogId catId; + DumpableObject *dobj; + + catId.tableoid = ExtensionRelationId; + catId.oid = oid; + dobj = findObjectByCatalogId(catId); + Assert(dobj == NULL || dobj->objType == DO_EXTENSION); + return (ExtensionInfo *) dobj; } /* * findPublicationByOid - * finds the entry (in pubinfo) of the publication with the given oid + * finds the DumpableObject for the publication with the given oid * returns NULL if not found */ PublicationInfo * findPublicationByOid(Oid oid) { - return (PublicationInfo *) findObjectByOid(oid, pubinfoindex, numPublications); + CatalogId catId; + DumpableObject *dobj; + + catId.tableoid = PublicationRelationId; + catId.oid = oid; + dobj = findObjectByCatalogId(catId); + Assert(dobj == NULL || dobj->objType == DO_PUBLICATION); + return (PublicationInfo *) dobj; } -/* - * findIndexByOid - * find the entry of the index with the given oid - * - * This one's signature is different from the previous ones because we lack a - * global array of all indexes, so caller must pass their array as argument. - */ -static IndxInfo * -findIndexByOid(Oid oid, DumpableObject **idxinfoindex, int numIndexes) -{ - return (IndxInfo *) findObjectByOid(oid, idxinfoindex, numIndexes); -} /* - * setExtensionMembership - * accept and save data about which objects belong to extensions + * recordExtensionMembership + * Record that the object identified by the given catalog ID + * belongs to the given extension */ void -setExtensionMembership(ExtensionMemberId *extmems, int nextmems) +recordExtensionMembership(CatalogId catId, ExtensionInfo *ext) { - /* Sort array in preparation for binary searches */ - if (nextmems > 1) - qsort((void *) extmems, nextmems, sizeof(ExtensionMemberId), - ExtensionMemberIdCompare); - /* And save */ - extmembers = extmems; - numextmembers = nextmems; + CatalogIdMapEntry *entry; + bool found; + + /* CatalogId hash table must exist, if we have an ExtensionInfo */ + Assert(catalogIdHash != NULL); + + /* Add reference to CatalogId hash */ + entry = catalogid_insert(catalogIdHash, catId, &found); + if (!found) + { + entry->dobj = NULL; + entry->ext = NULL; + } + Assert(entry->ext == NULL); + entry->ext = ext; } /* @@ -1048,56 +987,15 @@ setExtensionMembership(ExtensionMemberId *extmems, int nextmems) ExtensionInfo * findOwningExtension(CatalogId catalogId) { - ExtensionMemberId *low; - ExtensionMemberId *high; + CatalogIdMapEntry *entry; - /* - * We could use bsearch() here, but the notational cruft of calling - * bsearch is nearly as bad as doing it ourselves; and the generalized - * bsearch function is noticeably slower as well. - */ - if (numextmembers <= 0) - return NULL; - low = extmembers; - high = extmembers + (numextmembers - 1); - while (low <= high) - { - ExtensionMemberId *middle; - int difference; - - middle = low + (high - low) / 2; - /* comparison must match ExtensionMemberIdCompare, below */ - difference = oidcmp(middle->catId.oid, catalogId.oid); - if (difference == 0) - difference = oidcmp(middle->catId.tableoid, catalogId.tableoid); - if (difference == 0) - return middle->ext; - else if (difference < 0) - low = middle + 1; - else - high = middle - 1; - } - return NULL; -} + if (catalogIdHash == NULL) + return NULL; /* no objects exist yet */ -/* - * qsort comparator for ExtensionMemberIds - */ -static int -ExtensionMemberIdCompare(const void *p1, const void *p2) -{ - const ExtensionMemberId *obj1 = (const ExtensionMemberId *) p1; - const ExtensionMemberId *obj2 = (const ExtensionMemberId *) p2; - int cmpval; - - /* - * Compare OID first since it's usually unique, whereas there will only be - * a few distinct values of tableoid. - */ - cmpval = oidcmp(obj1->catId.oid, obj2->catId.oid); - if (cmpval == 0) - cmpval = oidcmp(obj1->catId.tableoid, obj2->catId.tableoid); - return cmpval; + entry = catalogid_lookup(catalogIdHash, catalogId); + if (entry == NULL) + return NULL; + return entry->ext; } diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 195800ab141..df7470feb2d 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -238,6 +238,7 @@ typedef struct Archive typedef struct { + /* Note: this struct must not contain any unused bytes */ Oid tableoid; Oid oid; } CatalogId; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 4f9f6f985f9..2117c9f82b8 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -6135,6 +6135,7 @@ getTypeStorageOptions(Archive *fout, int *numTypes) int i; PQExpBuffer query = createPQExpBuffer(); TypeStorageOptions *tstorageoptions; + int i_tableoid; int i_oid; int i_typname; int i_typnamespace; @@ -6152,17 +6153,18 @@ getTypeStorageOptions(Archive *fout, int *numTypes) * when dumping the type. */ appendPQExpBuffer(query, "SELECT " - " CASE WHEN t.oid > 10000 OR substring(t.typname from 1 for 1) = '_' " - " THEN quote_ident(t.typname) " - " ELSE pg_catalog.format_type(t.oid, NULL) " - " END as typname " - ", t.oid AS oid" - ", t.typnamespace AS typnamespace" - ", (%s typowner) as rolname" - ", array_to_string(a.typoptions, ', ') AS typoptions " - " FROM pg_type AS t " - " INNER JOIN pg_catalog.pg_type_encoding a ON a.typid = t.oid" - " WHERE t.typisdefined = 't'", username_subquery); + "CASE WHEN t.oid > 10000 OR substring(t.typname from 1 for 1) = '_' " + "THEN quote_ident(t.typname) " + "ELSE pg_catalog.format_type(t.oid, NULL) " + "END as typname, " + "t.tableoid as tableoid, " + "t.oid AS oid, " + "t.typnamespace AS typnamespace, " + "(%s typowner) as rolname, " + "array_to_string(a.typoptions, ', ') AS typoptions " + "FROM pg_type t " + "JOIN pg_catalog.pg_type_encoding a ON a.typid = t.oid " + "WHERE t.typisdefined = 't'", username_subquery); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -6170,8 +6172,9 @@ getTypeStorageOptions(Archive *fout, int *numTypes) tstorageoptions = (TypeStorageOptions *) pg_malloc(ntups * sizeof(TypeStorageOptions)); - i_typname = PQfnumber(res, "typname"); + i_tableoid = PQfnumber(res, "tableoid"); i_oid = PQfnumber(res, "oid"); + i_typname = PQfnumber(res, "typname"); i_typnamespace = PQfnumber(res, "typnamespace"); i_typoptions = PQfnumber(res, "typoptions"); i_rolname = PQfnumber(res, "rolname"); @@ -6179,9 +6182,10 @@ getTypeStorageOptions(Archive *fout, int *numTypes) for (i = 0; i < ntups; i++) { tstorageoptions[i].dobj.objType = DO_TYPE_STORAGE_OPTIONS; + tstorageoptions[i].dobj.catId.tableoid = atooid(PQgetvalue(res, i, i_tableoid)); + tstorageoptions[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid)); AssignDumpId(&tstorageoptions[i].dobj); tstorageoptions[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_typname)); - tstorageoptions[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid)); tstorageoptions[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_typnamespace))); tstorageoptions[i].typoptions = pg_strdup(PQgetvalue(res, i, i_typoptions)); tstorageoptions[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); @@ -20336,12 +20340,10 @@ getExtensionMembership(Archive *fout, ExtensionInfo extinfo[], PQExpBuffer query; PGresult *res; int ntups, - nextmembers, i; int i_classid, i_objid, i_refobjid; - ExtensionMemberId *extmembers; ExtensionInfo *ext; /* Nothing to do if no extensions */ @@ -20366,12 +20368,7 @@ getExtensionMembership(Archive *fout, ExtensionInfo extinfo[], i_objid = PQfnumber(res, "objid"); i_refobjid = PQfnumber(res, "refobjid"); - extmembers = (ExtensionMemberId *) pg_malloc(ntups * sizeof(ExtensionMemberId)); - nextmembers = 0; - /* - * Accumulate data into extmembers[]. - * * Since we ordered the SELECT by referenced ID, we can expect that * multiple entries for the same extension will appear together; this * saves on searches. @@ -20398,16 +20395,11 @@ getExtensionMembership(Archive *fout, ExtensionInfo extinfo[], continue; } - extmembers[nextmembers].catId = objId; - extmembers[nextmembers].ext = ext; - nextmembers++; + recordExtensionMembership(objId, ext); } PQclear(res); - /* Remember the data for use later */ - setExtensionMembership(extmembers, nextmembers); - destroyPQExpBuffer(query); } diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 8a79ea2d601..fec9f6fde8c 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -703,17 +703,6 @@ typedef struct _SubscriptionInfo char *subpublications; } SubscriptionInfo; -/* - * We build an array of these with an entry for each object that is an - * extension member according to pg_depend. - */ -typedef struct _extensionMemberId -{ - CatalogId catId; /* tableoid+oid of some member object */ - ExtensionInfo *ext; /* owning extension */ -} ExtensionMemberId; - -extern const char *EXT_PARTITION_NAME_POSTFIX; /* * common utility functions */ @@ -739,7 +728,7 @@ extern NamespaceInfo *findNamespaceByOid(Oid oid); extern ExtensionInfo *findExtensionByOid(Oid oid); extern PublicationInfo *findPublicationByOid(Oid oid); -extern void setExtensionMembership(ExtensionMemberId *extmems, int nextmems); +extern void recordExtensionMembership(CatalogId catId, ExtensionInfo *ext); extern ExtensionInfo *findOwningExtension(CatalogId catalogId); extern void parseOidArray(const char *str, Oid *array, int arraysize); From af2b9cecb3b52efb0914f0b590de504b83baed67 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 24 Oct 2021 12:38:26 -0400 Subject: [PATCH 4/7] Fix minor memory leaks in pg_dump. I found these by running pg_dump under "valgrind --leak-check=full". The changes in flagInhIndexes() and getIndexes() replace allocation of an array of which we use only some elements by individual allocations of just the actually-needed objects. The previous coding wasted some memory, but more importantly it confused valgrind's leak tracking. collectComments() and collectSecLabels() remain major blots on the valgrind report, because they don't PQclear their query results, in order to avoid a lot of strdup's. That's a dubious tradeoff, but I'll leave it alone here; an upcoming patch will modify those functions enough to justify changing the tradeoff. (cherry-picked from 70bef494000e4dbbeca0f0a40347ca1747aea701) --- src/bin/pg_dump/common.c | 42 +++++++------- src/bin/pg_dump/pg_backup_archiver.c | 2 + src/bin/pg_dump/pg_dump.c | 85 ++++++++++++++++++---------- 3 files changed, 78 insertions(+), 51 deletions(-) diff --git a/src/bin/pg_dump/common.c b/src/bin/pg_dump/common.c index 85515df381a..e47f354cfdd 100644 --- a/src/bin/pg_dump/common.c +++ b/src/bin/pg_dump/common.c @@ -280,6 +280,8 @@ getSchemaData(Archive *fout, int *numTablesPtr) pg_log_info("reading subscriptions"); getSubscriptions(fout); + free(inhinfo); /* not needed any longer */ + *numTablesPtr = numTables; return tblinfo; } @@ -401,24 +403,20 @@ static void flagInhIndexes(Archive *fout, TableInfo tblinfo[], int numTables) { int i, - j, - k; + j; for (i = 0; i < numTables; i++) { - IndexAttachInfo *attachinfo; - if (!tblinfo[i].ispartition || tblinfo[i].numParents == 0) continue; Assert(tblinfo[i].numParents == 1); - attachinfo = (IndexAttachInfo *) - pg_malloc0(tblinfo[i].numIndexes * sizeof(IndexAttachInfo)); - for (j = 0, k = 0; j < tblinfo[i].numIndexes; j++) + for (j = 0; j < tblinfo[i].numIndexes; j++) { IndxInfo *index = &(tblinfo[i].indexes[j]); IndxInfo *parentidx; + IndexAttachInfo *attachinfo; if (index->parentidx == 0) continue; @@ -427,14 +425,16 @@ flagInhIndexes(Archive *fout, TableInfo tblinfo[], int numTables) if (parentidx == NULL) continue; - attachinfo[k].dobj.objType = DO_INDEX_ATTACH; - attachinfo[k].dobj.catId.tableoid = 0; - attachinfo[k].dobj.catId.oid = 0; - AssignDumpId(&attachinfo[k].dobj); - attachinfo[k].dobj.name = pg_strdup(index->dobj.name); - attachinfo[k].dobj.namespace = index->indextable->dobj.namespace; - attachinfo[k].parentIdx = parentidx; - attachinfo[k].partitionIdx = index; + attachinfo = (IndexAttachInfo *) pg_malloc(sizeof(IndexAttachInfo)); + + attachinfo->dobj.objType = DO_INDEX_ATTACH; + attachinfo->dobj.catId.tableoid = 0; + attachinfo->dobj.catId.oid = 0; + AssignDumpId(&attachinfo->dobj); + attachinfo->dobj.name = pg_strdup(index->dobj.name); + attachinfo->dobj.namespace = index->indextable->dobj.namespace; + attachinfo->parentIdx = parentidx; + attachinfo->partitionIdx = index; /* * We must state the DO_INDEX_ATTACH object's dependencies @@ -451,17 +451,15 @@ flagInhIndexes(Archive *fout, TableInfo tblinfo[], int numTables) * will not try to run the ATTACH concurrently with other * operations on those tables. */ - addObjectDependency(&attachinfo[k].dobj, index->dobj.dumpId); - addObjectDependency(&attachinfo[k].dobj, parentidx->dobj.dumpId); - addObjectDependency(&attachinfo[k].dobj, + addObjectDependency(&attachinfo->dobj, index->dobj.dumpId); + addObjectDependency(&attachinfo->dobj, parentidx->dobj.dumpId); + addObjectDependency(&attachinfo->dobj, index->indextable->dobj.dumpId); - addObjectDependency(&attachinfo[k].dobj, + addObjectDependency(&attachinfo->dobj, parentidx->indextable->dobj.dumpId); /* keep track of the list of partitions in the parent index */ - simple_ptr_list_append(&parentidx->partattaches, &attachinfo[k].dobj); - - k++; + simple_ptr_list_append(&parentidx->partattaches, &attachinfo->dobj); } } } diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index a974c28e82c..b2ae4aaf046 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -3450,6 +3450,8 @@ _selectTableAccessMethod(ArchiveHandle *AH, const char *tableam) destroyPQExpBuffer(cmd); + if (AH->currTableAm) + free(AH->currTableAm); AH->currTableAm = pg_strdup(want); } diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 2117c9f82b8..e9bee1cba52 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -8255,7 +8255,6 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) PQExpBuffer query = createPQExpBuffer(); PGresult *res; IndxInfo *indxinfo; - ConstraintInfo *constrinfo; int i_tableoid, i_oid, i_indexname, @@ -8486,7 +8485,6 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) tbinfo->indexes = indxinfo = (IndxInfo *) pg_malloc(ntups * sizeof(IndxInfo)); - constrinfo = (ConstraintInfo *) pg_malloc(ntups * sizeof(ConstraintInfo)); tbinfo->numIndexes = ntups; for (j = 0; j < ntups; j++) @@ -8526,28 +8524,31 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) * If we found a constraint matching the index, create an * entry for it. */ - constrinfo[j].dobj.objType = DO_CONSTRAINT; - constrinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_contableoid)); - constrinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_conoid)); - AssignDumpId(&constrinfo[j].dobj); - constrinfo[j].dobj.dump = tbinfo->dobj.dump; - constrinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_conname)); - constrinfo[j].dobj.namespace = tbinfo->dobj.namespace; - constrinfo[j].contable = tbinfo; - constrinfo[j].condomain = NULL; - constrinfo[j].contype = contype; + ConstraintInfo *constrinfo; + + constrinfo = (ConstraintInfo *) pg_malloc(sizeof(ConstraintInfo)); + constrinfo->dobj.objType = DO_CONSTRAINT; + constrinfo->dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_contableoid)); + constrinfo->dobj.catId.oid = atooid(PQgetvalue(res, j, i_conoid)); + AssignDumpId(&constrinfo->dobj); + constrinfo->dobj.dump = tbinfo->dobj.dump; + constrinfo->dobj.name = pg_strdup(PQgetvalue(res, j, i_conname)); + constrinfo->dobj.namespace = tbinfo->dobj.namespace; + constrinfo->contable = tbinfo; + constrinfo->condomain = NULL; + constrinfo->contype = contype; if (contype == 'x') - constrinfo[j].condef = pg_strdup(PQgetvalue(res, j, i_condef)); + constrinfo->condef = pg_strdup(PQgetvalue(res, j, i_condef)); else - constrinfo[j].condef = NULL; - constrinfo[j].confrelid = InvalidOid; - constrinfo[j].conindex = indxinfo[j].dobj.dumpId; - constrinfo[j].condeferrable = *(PQgetvalue(res, j, i_condeferrable)) == 't'; - constrinfo[j].condeferred = *(PQgetvalue(res, j, i_condeferred)) == 't'; - constrinfo[j].conislocal = true; - constrinfo[j].separate = true; - - indxinfo[j].indexconstraint = constrinfo[j].dobj.dumpId; + constrinfo->condef = NULL; + constrinfo->confrelid = InvalidOid; + constrinfo->conindex = indxinfo[j].dobj.dumpId; + constrinfo->condeferrable = *(PQgetvalue(res, j, i_condeferrable)) == 't'; + constrinfo->condeferred = *(PQgetvalue(res, j, i_condeferred)) == 't'; + constrinfo->conislocal = true; + constrinfo->separate = true; + + indxinfo[j].indexconstraint = constrinfo->dobj.dumpId; } else { @@ -13285,6 +13286,7 @@ dumpFunc(Archive *fout, const FuncInfo *finfo) char *funcsig; /* identity signature */ char *funcfullsig = NULL; /* full signature */ char *funcsig_tag; + char *qual_funcsig; char *proretset; char *prosrc; char *probin; @@ -13623,15 +13625,39 @@ dumpFunc(Archive *fout, const FuncInfo *finfo) funcsig_tag = format_function_signature(fout, finfo, false); + if (prokind[0] == PROKIND_PROCEDURE) + keyword = "PROCEDURE"; + else + { + configitems = NULL; + nconfigitems = 0; + } + + if (funcargs) + { + /* 8.4 or later; we rely on server-side code for most of the work */ + funcfullsig = format_function_arguments(finfo, funcargs, false); + funcsig = format_function_arguments(finfo, funciargs, false); + } + else + /* pre-8.4, do it ourselves */ + funcsig = format_function_arguments_old(fout, + finfo, nallargs, allargtypes, + argmodes, argnames); + + funcsig_tag = format_function_signature(fout, finfo, false); + + qual_funcsig = psprintf("%s.%s", + fmtId(finfo->dobj.namespace->dobj.name), + funcsig); + if (prokind[0] == PROKIND_PROCEDURE) keyword = "PROCEDURE"; else keyword = "FUNCTION"; /* works for window functions too */ - appendPQExpBuffer(delqry, "DROP %s %s.%s;\n", - keyword, - fmtId(finfo->dobj.namespace->dobj.name), - funcsig); + appendPQExpBuffer(delqry, "DROP %s %s;\n", + keyword, qual_funcsig); appendPQExpBuffer(q, "CREATE %s %s.%s", keyword, @@ -13829,9 +13855,7 @@ dumpFunc(Archive *fout, const FuncInfo *finfo) append_depends_on_extension(fout, q, &finfo->dobj, "pg_catalog.pg_proc", keyword, - psprintf("%s.%s", - fmtId(finfo->dobj.namespace->dobj.name), - funcsig)); + qual_funcsig); if (dopt->binary_upgrade) binary_upgrade_extension_member(q, &finfo->dobj, @@ -13876,6 +13900,7 @@ dumpFunc(Archive *fout, const FuncInfo *finfo) if (funcfullsig) free(funcfullsig); free(funcsig_tag); + free(qual_funcsig); if (allargtypes) free(allargtypes); if (argmodes) @@ -16499,6 +16524,8 @@ dumpForeignServer(Archive *fout, const ForeignServerInfo *srvinfo) srvinfo->rolname, srvinfo->dobj.catId, srvinfo->dobj.dumpId); + PQclear(res); + free(qsrvname); destroyPQExpBuffer(q); From b947094072aedd3f98c3e3198ddce30aaf354545 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 2 Dec 2021 16:46:28 +0100 Subject: [PATCH 5/7] pg_dump: Add missing relkind case Checking for RELKIND_MATVIEW was forgotten in guessConstraintInheritance(). This isn't a live problem, since it is checked in flagInhTables() which relkinds can have parents, and those entries will have numParents==0 after that. But after discussion it was felt that this place should be kept consistent with flagInhTables() and flagInhAttrs(). Reviewed-by: Michael Paquier Discussion: https://www.postgresql.org/message-id/flat/a574c8f1-9c84-93ad-a9e5-65233d6fc00f@enterprisedb.com (cherry picked from commit a22d6a2cb62c4fc6d7c4b077d8014fd4ffaec426) --- src/bin/pg_dump/pg_dump.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index e9bee1cba52..2d77da17d20 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -3155,9 +3155,10 @@ guessConstraintInheritance(TableInfo *tblinfo, int numTables) TableInfo **parents; TableInfo *parent; - /* Sequences and views never have parents */ + /* Some kinds never have parents */ if (tbinfo->relkind == RELKIND_SEQUENCE || - tbinfo->relkind == RELKIND_VIEW) + tbinfo->relkind == RELKIND_VIEW || + tbinfo->relkind == RELKIND_MATVIEW) continue; /* Don't bother computing anything for non-target tables, either */ From 2c8a3e844df552d530a7f75f657c610620cb856c Mon Sep 17 00:00:00 2001 From: Brent Doil Date: Tue, 26 Jul 2022 23:47:31 -0400 Subject: [PATCH 6/7] pg_dump: Remove unused TypeCache struct Authored-by: Brent Doil --- src/bin/pg_dump/pg_dump.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index fec9f6fde8c..a0bf7f4ce86 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -198,16 +198,6 @@ typedef struct _typeInfo struct _constraintInfo *domChecks; } TypeInfo; -typedef struct _typeCache -{ - DumpableObject dobj; - - Oid typnsp; - - Oid arraytypoid; - char *arraytypname; - Oid arraytypnsp; -} TypeCache; typedef struct _typeStorageOptions { From 2dc796aa5d2532f097d489b88081685c5d1eaa71 Mon Sep 17 00:00:00 2001 From: Brent Doil Date: Fri, 22 Jul 2022 18:22:07 -0400 Subject: [PATCH 7/7] pg_dump: Lock all interesting tables in single statement. This reduces the potentially significant command and communication overhead between frontend and backend when sending a LOCK TABLE query for each table. Discussion: https://www.postgresql.org/message-id/20120530.180620.600165924826262795.t-ishii%40sraoss.co.jp Authored-by: Brent Doil --- src/bin/pg_dump/pg_dump.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 2d77da17d20..cb4b6e1db4c 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -7256,6 +7256,7 @@ getTables(Archive *fout, int *numTables) int i; PQExpBuffer query = createPQExpBuffer(); TableInfo *tblinfo; + bool lockTableDumped = false; int i_reltableoid; int i_reloid; int i_relname; @@ -7963,6 +7964,8 @@ getTables(Archive *fout, int *numTables) ExecuteSqlStatement(fout, query->data); } + resetPQExpBuffer(query); + for (i = 0; i < ntups; i++) { tblinfo[i].dobj.objType = DO_TABLE; @@ -8098,6 +8101,11 @@ getTables(Archive *fout, int *numTables) * * We only need to lock the table for certain components; see * pg_dump.h + * + * GPDB: Build a single LOCK TABLE statement to lock all interesting tables. + * This is more performant than issuing a separate LOCK TABLE statement for each table, + * with considerable savings in FE/BE overhead. It does come at the cost of some increased + * memory usage in both FE and BE, which we will be able to tolerate. */ /* GPDB_14_MERGE_FIXME: GPDB_96_MERGE_FIXME: Is the parrelid check still needed? */ if (tblinfo[i].dobj.dump && @@ -8106,11 +8114,15 @@ getTables(Archive *fout, int *numTables) tblinfo[i].parrelid == 0 && (tblinfo[i].dobj.dump & DUMP_COMPONENTS_REQUIRING_LOCK)) { - resetPQExpBuffer(query); - appendPQExpBuffer(query, - "LOCK TABLE %s IN ACCESS SHARE MODE", - fmtQualifiedDumpable(&tblinfo[i])); - ExecuteSqlStatement(fout, query->data); + if (!lockTableDumped) + appendPQExpBuffer(query, + "LOCK TABLE %s ", + fmtQualifiedDumpable(&tblinfo[i])); + else + appendPQExpBuffer(query, + ",%s ", + fmtQualifiedDumpable(&tblinfo[i])); + lockTableDumped = true; } /* Emit notice if join for owner failed */ @@ -8118,6 +8130,13 @@ getTables(Archive *fout, int *numTables) pg_log_warning("owner of table \"%s\" appears to be invalid", tblinfo[i].dobj.name); } + /* Are there any tables to lock? */ + if (lockTableDumped) + { + appendPQExpBuffer(query, + "IN ACCESS SHARE MODE"); + ExecuteSqlStatement(fout, query->data); + } if (dopt->lockWaitTimeout) {