From fb05f3ce83d225dd0f39f8860ce04082753e9e98 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 22 Feb 2014 13:38:06 -0500 Subject: [PATCH] pg_basebackup: Add support for relocating tablespaces Tablespaces can be relocated in plain backup mode by specifying one or more -T olddir=newdir options. Author: Steeve Lennmark Reviewed-by: Peter Eisentraut --- doc/src/sgml/ref/pg_basebackup.sgml | 46 ++++++- src/bin/pg_basebackup/pg_basebackup.c | 166 +++++++++++++++++++++++++- 2 files changed, 204 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index c379df546c..ea2233123e 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -202,6 +202,33 @@ PostgreSQL documentation + + + + + + Relocate the tablespace in directory olddir + to newdir during the backup. To be + effective, olddir must exactly match the + path specification of the tablespace as it is currently defined. (But + it is not an error if there is no tablespace + in olddir contained in the backup.) + Both olddir + and newdir must be absolute paths. If a + path happens to contain a = sign, escape it with a + backslash. This option can be specified multiple times for multiple + tablespaces. See examples below. + + + + If a tablespace is relocated in this way, the symbolic links inside + the main data directory are updated to point to the new location. So + the new data directory is ready to be used for a new server instance + with all tablespaces in the updated locations. + + + + @@ -528,9 +555,13 @@ PostgreSQL documentation - The way PostgreSQL manages tablespaces, the path - for all additional tablespaces must be identical whenever a backup is - restored. The main data directory, however, is relocatable to any location. + Tablespaces will in plain format by default be backed up to the same path + they have on the server, unless the + option --tablespace-mapping is used. Without + this option, running a plain format base backup on the same host as the + server will not work if tablespaces are in use, because the backup would + have to be written to the same directory locations as the original + tablespaces. @@ -570,6 +601,15 @@ PostgreSQL documentation (This command will fail if there are multiple tablespaces in the database.) + + + To create a backup of a local database where the tablespace in + /opt/ts is relocated + to ./backup/ts: + +$ pg_basebackup -D backup/data -T /opt/ts=$(pwd)/backup/ts + + diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 3d155e8907..9d7a1e38ad 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -35,8 +35,24 @@ #include "streamutil.h" +#define atooid(x) ((Oid) strtoul((x), NULL, 10)) + +typedef struct TablespaceListCell +{ + struct TablespaceListCell *next; + char old_dir[MAXPGPATH]; + char new_dir[MAXPGPATH]; +} TablespaceListCell; + +typedef struct TablespaceList +{ + TablespaceListCell *head; + TablespaceListCell *tail; +} TablespaceList; + /* Global options */ static char *basedir = NULL; +static TablespaceList tablespace_dirs = {NULL, NULL}; static char *xlog_dir = ""; static char format = 'p'; /* p(lain)/t(ar) */ static char *label = "pg_basebackup base backup"; @@ -90,6 +106,10 @@ static void BaseBackup(void); static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, bool segment_finished); +static const char *get_tablespace_mapping(const char *dir); +static void update_tablespace_symlink(Oid oid, const char *old_dir); +static void tablespace_list_append(const char *arg); + static void disconnect_and_exit(int code) { @@ -110,6 +130,77 @@ static void disconnect_and_exit(int code) } +/* + * Split argument into old_dir and new_dir and append to tablespace mapping + * list. + */ +static void +tablespace_list_append(const char *arg) +{ + TablespaceListCell *cell = (TablespaceListCell *) pg_malloc0(sizeof(TablespaceListCell)); + char *dst; + char *dst_ptr; + const char *arg_ptr; + + dst_ptr = dst = cell->old_dir; + for (arg_ptr = arg; *arg_ptr; arg_ptr++) + { + if (dst_ptr - dst >= MAXPGPATH) + { + fprintf(stderr, _("%s: directory name too long\n"), progname); + exit(1); + } + + if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') + ; /* skip backslash escaping = */ + else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) + { + if (*cell->new_dir) + { + fprintf(stderr, _("%s: multiple \"=\" signs in tablespace mapping\n"), progname); + exit(1); + } + else + dst = dst_ptr = cell->new_dir; + } + else + *dst_ptr++ = *arg_ptr; + } + + if (!*cell->old_dir || !*cell->new_dir) + { + fprintf(stderr, + _("%s: invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"\n"), + progname, arg); + exit(1); + } + + /* This check isn't absolutely necessary. But all tablespaces are created + * with absolute directories, so specifying a non-absolute path here would + * just never match, possibly confusing users. It's also good to be + * consistent with the new_dir check. */ + if (!is_absolute_path(cell->old_dir)) + { + fprintf(stderr, _("%s: old directory not absolute in tablespace mapping: %s\n"), + progname, cell->old_dir); + exit(1); + } + + if (!is_absolute_path(cell->new_dir)) + { + fprintf(stderr, _("%s: new directory not absolute in tablespace mapping: %s\n"), + progname, cell->new_dir); + exit(1); + } + + if (tablespace_dirs.tail) + tablespace_dirs.tail->next = cell; + else + tablespace_dirs.head = cell; + tablespace_dirs.tail = cell; +} + + #ifdef HAVE_LIBZ static const char * get_gz_error(gzFile gzf) @@ -137,6 +228,8 @@ usage(void) printf(_(" -F, --format=p|t output format (plain (default), tar)\n")); printf(_(" -R, --write-recovery-conf\n" " write recovery.conf after backup\n")); + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n" + " relocate tablespace in OLDDIR to NEWDIR\n")); printf(_(" -x, --xlog include required WAL files in backup (fetch mode)\n")); printf(_(" -X, --xlog-method=fetch|stream\n" " include required WAL files with specified method\n")); @@ -899,6 +992,52 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) PQfreemem(copybuf); } + +/* + * Retrieve tablespace path, either relocated or original depending on whether + * -T was passed or not. + */ +static const char * +get_tablespace_mapping(const char *dir) +{ + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(dir, cell->old_dir) == 0) + return cell->new_dir; + + return dir; +} + + +/* + * Update symlinks to reflect relocated tablespace. + */ +static void +update_tablespace_symlink(Oid oid, const char *old_dir) +{ + const char *new_dir = get_tablespace_mapping(old_dir); + + if (strcmp(old_dir, new_dir) != 0) + { + char *linkloc = psprintf("%s/pg_tblspc/%d", basedir, oid); + + if (unlink(linkloc) != 0 && errno != ENOENT) + { + fprintf(stderr, _("%s: could not remove symbolic link \"%s\": %s"), + progname, linkloc, strerror(errno)); + disconnect_and_exit(1); + } + if (symlink(new_dir, linkloc) != 0) + { + fprintf(stderr, _("%s: could not create symbolic link \"%s\": %s"), + progname, linkloc, strerror(errno)); + disconnect_and_exit(1); + } + } +} + + /* * Receive a tar format stream from the connection to the server, and unpack * the contents of it into a directory. Only files, directories and @@ -906,8 +1045,7 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) * * If the data is for the main data directory, it will be restored in the * specified directory. If it's for another tablespace, it will be restored - * in the original directory, since relocation of tablespaces is not - * supported. + * in the original or mapped directory. */ static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) @@ -923,7 +1061,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) if (basetablespace) strlcpy(current_path, basedir, sizeof(current_path)); else - strlcpy(current_path, PQgetvalue(res, rownum, 1), sizeof(current_path)); + strlcpy(current_path, get_tablespace_mapping(PQgetvalue(res, rownum, 1)), sizeof(current_path)); /* * Get the COPY data @@ -1503,7 +1641,10 @@ BaseBackup(void) * we do anything anyway. */ if (format == 'p' && !PQgetisnull(res, i, 1)) - verify_dir_is_empty_or_create(PQgetvalue(res, i, 1)); + { + char *path = (char *) get_tablespace_mapping(PQgetvalue(res, i, 1)); + verify_dir_is_empty_or_create(path); + } } /* @@ -1545,6 +1686,17 @@ BaseBackup(void) progress_report(PQntuples(res), NULL, true); fprintf(stderr, "\n"); /* Need to move to next line */ } + + if (format == 'p' && tablespace_dirs.head != NULL) + { + for (i = 0; i < PQntuples(res); i++) + { + Oid tblspc_oid = atooid(PQgetvalue(res, i, 0)); + if (tblspc_oid) + update_tablespace_symlink(tblspc_oid, PQgetvalue(res, i, 1)); + } + } + PQclear(res); /* @@ -1696,6 +1848,7 @@ main(int argc, char **argv) {"format", required_argument, NULL, 'F'}, {"checkpoint", required_argument, NULL, 'c'}, {"write-recovery-conf", no_argument, NULL, 'R'}, + {"tablespace-mapping", required_argument, NULL, 'T'}, {"xlog", no_argument, NULL, 'x'}, {"xlog-method", required_argument, NULL, 'X'}, {"gzip", no_argument, NULL, 'z'}, @@ -1735,7 +1888,7 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "D:F:RxX:l:zZ:d:c:h:p:U:s:wWvP", + while ((c = getopt_long(argc, argv, "D:F:RT:xX:l:zZ:d:c:h:p:U:s:wWvP", long_options, &option_index)) != -1) { switch (c) @@ -1759,6 +1912,9 @@ main(int argc, char **argv) case 'R': writerecoveryconf = true; break; + case 'T': + tablespace_list_append(optarg); + break; case 'x': if (includewal) {