From 048d148fe63102fafb2336ab5439c950dea7f692 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sun, 23 Jan 2011 12:21:23 +0100 Subject: [PATCH] Add pg_basebackup tool for streaming base backups This tool makes it possible to do the pg_start_backup/ copy files/pg_stop_backup step in a single command. There are still some steps to be done before this is a complete backup solution, such as the ability to stream the required WAL logs, but it's still usable, and could do with some buildfarm coverage. In passing, make the checkpoint request optionally fast instead of hardcoding it. Magnus Hagander, reviewed by Fujii Masao and Dimitri Fontaine --- doc/src/sgml/backup.sgml | 10 + doc/src/sgml/config.sgml | 3 +- doc/src/sgml/protocol.sgml | 11 +- doc/src/sgml/ref/allfiles.sgml | 1 + doc/src/sgml/ref/pg_basebackup.sgml | 397 ++++++++++++ doc/src/sgml/reference.sgml | 1 + src/backend/replication/basebackup.c | 19 +- src/backend/replication/repl_gram.y | 11 +- src/backend/replication/repl_scanner.l | 1 + src/backend/replication/walsender.c | 2 +- src/bin/Makefile | 2 +- src/bin/pg_basebackup/Makefile | 38 ++ src/bin/pg_basebackup/nls.mk | 5 + src/bin/pg_basebackup/pg_basebackup.c | 1035 ++++++++++++++++++++++++++++++++ src/include/replication/basebackup.h | 2 +- src/include/replication/replnodes.h | 1 + src/tools/msvc/Mkvcbuild.pm | 2 + 17 files changed, 1528 insertions(+), 13 deletions(-) create mode 100644 doc/src/sgml/ref/pg_basebackup.sgml create mode 100644 src/bin/pg_basebackup/Makefile create mode 100644 src/bin/pg_basebackup/nls.mk create mode 100644 src/bin/pg_basebackup/pg_basebackup.c diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index db7c834914..c14ae43062 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -813,6 +813,16 @@ SELECT pg_stop_backup(); + You can also use the tool to take + the backup, instead of manually copying the files. This tool will take + care of the pg_start_backup(), copy and + pg_stop_backup() steps automatically, and transfers the + backup over a regular PostgreSQL connection + using the replication protocol, instead of requiring filesystem level + access. + + + Some file system backup tools emit warnings or errors if the files they are trying to copy change while the copy proceeds. When taking a base backup of an active database, this situation is normal diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 570c7c3b7d..bbfe86a692 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1921,7 +1921,8 @@ SET ENABLE_SEQSCAN TO OFF; Specifies the maximum number of concurrent connections from standby - servers (i.e., the maximum number of simultaneously running WAL sender + servers or streaming base backup clients (i.e., the maximum number of + simultaneously running WAL sender processes). The default is zero. This parameter can only be set at server start. wal_level must be set to archive or hot_standby to allow connections from standby servers. diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 76c062fd51..73f26b432d 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1460,7 +1460,7 @@ The commands accepted in walsender mode are: - BASE_BACKUP [LABEL 'label'] [PROGRESS] + BASE_BACKUP [LABEL 'label'] [PROGRESS] [FAST] Instructs the server to start streaming a base backup. @@ -1496,6 +1496,15 @@ The commands accepted in walsender mode are: + + + FAST + + + Request a fast checkpoint. + + + diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index f40fa9dd8b..c44d11ef91 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -160,6 +160,7 @@ Complete list of usable sgml source files in this directory. + diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml new file mode 100644 index 0000000000..321c8cade1 --- /dev/null +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -0,0 +1,397 @@ + + + + + pg_basebackup + 1 + Application + + + + pg_basebackup + take a base backup of a PostgreSQL cluster + + + + pg_basebackup + + + + + pg_basebackup + option + + + + + + Description + + + pg_basebackup is used to take base backups of + a running PostgreSQL database cluster. These + are taken without affecting other clients to the database, and can be used + both for point-in-time recovery (see ) + and as the starting point for a log shipping or streaming replication standby + servers (see ). + + + + pg_basebackup makes a binary copy of the database + cluster files, while making sure the system is automatically put in and + out of backup mode automatically. Backups are always taken of the entire + database cluster, it is not possible to back up individual databases or + database objects. For individual database backups, a tool such as + must be used. + + + + The backup is made over a regular PostgreSQL + connection, and uses the replication protocol. The connection must be + made with a user having REPLICATION permissions (see + ), and the user must be granted explicit + permissions in pg_hba.conf. The server must also + be configured with set high enough + to leave at least one session available for the backup. + + + + Only one backup can be concurrently active in + PostgreSQL, meaning that only one instance of + pg_basebackup can run at the same time + against a single database cluster. + + + + + Options + + + The following command-line options control the location and format of the + output. + + + + + + + + Directory to write the output to. + + + When the backup is in tar mode, and the directory is specified as + - (dash), the tar file will be written to + stdout. + + + This parameter is required. + + + + + + + + + + Selects the format for the output. format + can be one of the following: + + + + p + plain + + + Write the output as plain files, with the same layout as the + current data directory and tablespaces. When the cluster has + no additional tablespaces, the whole database will be placed in + the target directory. If the cluster contains additional + tablespaces, the main data directory will be placed in the + target directory, but all other tablespaces will be placed + in the same absolute path as they have on the server. + + + This is the default format. + + + + + + t + tar + + + Write the output as tar files in the target directory. The main + data directory will be written to a file named + base.tar, and all other tablespaces will + be named after the tablespace oid. + + + If the value - (dash) is specified as + target directory, the tar contents will be written to + standard output, suitable for piping to for example + gzip. This is only possible if + the cluster has no additional tablespaces. + + + + + + + + + + + + + + Enables gzip compression of tar file output. Compression is only + available when generating tar files, and is not available when sending + output to standard output. + + + + + + + The following command-line options control the generation of the + backup and the running of the program. + + + + + + + + Sets checkpoint mode to fast or spread (default). + + + + + + + + + + Sets the label for the backup. If none is specified, a default value of + pg_basebackup base backup will be used. + + + + + + + + + + Enables progress reporting. Turning this on will deliver an approximate + progress report during the backup. Since the database may change during + the backup, this is only an approximation and may not end at exactly + 100%. + + + When this is enabled, the backup will start by enumerating the size of + the entire database, and then go back and send the actual contents. + This may make the backup take slightly longer, and in particular it + will take longer before the first data is sent. + + + + + + + + + + Enables verbose mode. Will output some extra steps during startup and + shutdown, as well as show the exact filename that is currently being + processed if progress reporting is also enabled. + + + + + + + + + The following command-line options control the database connection parameters. + + + + + + + + Specifies the host name of the machine on which the server is + running. If the value begins with a slash, it is used as the + directory for the Unix domain socket. The default is taken + from the PGHOST environment variable, if set, + else a Unix domain socket connection is attempted. + + + + + + + + + + Specifies the TCP port or local Unix domain socket file + extension on which the server is listening for connections. + Defaults to the PGPORT environment variable, if + set, or a compiled-in default. + + + + + + + + + + User name to connect as. + + + + + + + + + + Never issue a password prompt. If the server requires + password authentication and a password is not available by + other means such as a .pgpass file, the + connection attempt will fail. This option can be useful in + batch jobs and scripts where no user is present to enter a + password. + + + + + + + + + + Force pg_basebackup to prompt for a + password before connecting to a database. + + + + This option is never essential, since + pg_bsaebackup will automatically prompt + for a password if the server demands password authentication. + However, pg_basebackup will waste a + connection attempt finding out that the server wants a password. + In some cases it is worth typing + + + + + + + Other, less commonly used, parameters are also available: + + + + + + + + Print the pg_basebackup version and exit. + + + + + + + + + + Show help about pg_basebackup command line + arguments, and exit. + + + + + + + + + + + Environment + + + This utility, like most other PostgreSQL utilities, + uses the environment variables supported by libpq + (see ). + + + + + + Notes + + + The backup will include all files in the data directory and tablespaces, + including the configuration files and any additional files placed in the + directory by third parties. Only regular files and directories are allowed + in the data directory, no symbolic links or special device files. + + + + The way PostgreSQL manages tablespaces, the path + for all additional tablespaces must be identical whenever a backup is + restored. The main data directory, however, is relocatable to any location. + + + + + Examples + + + To create a base backup of the server at mydbserver + and store it in the local directory + /usr/local/pgsql/data: + + $ pg_basebackup -h mydbserver -D /usr/local/pgsql/data + + + + + To create a backup of the local server with one maximum compressed + tar file for each tablespace, and store it in the directory + backup, showing a progress report while running: + + $ pg_basebackup -D backup -Ft -Z9 -P + + + + + To create a backup of a single-tablespace local database and compress + this with bzip2: + + $ pg_basebackup -D - -Ft | bzip2 > backup.tar.bz2 + + (this command will fail if there are multiple tablespaces in the + database) + + + + + See Also + + + + + + + diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index 84babf61c0..6ee8e5bcff 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -202,6 +202,7 @@ &droplang; &dropuser; &ecpgRef; + &pgBasebackup; &pgConfig; &pgDump; &pgDumpall; diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index b4d5bbe412..943d80470b 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -40,7 +40,7 @@ static void send_int8_string(StringInfoData *buf, int64 intval); static void SendBackupHeader(List *tablespaces); static void SendBackupDirectory(char *location, char *spcoid); static void base_backup_cleanup(int code, Datum arg); -static void perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir); +static void perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir, bool fastcheckpoint); typedef struct { @@ -67,9 +67,9 @@ base_backup_cleanup(int code, Datum arg) * clobbered by longjmp" from stupider versions of gcc. */ static void -perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir) +perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir, bool fastcheckpoint) { - do_pg_start_backup(backup_label, true); + do_pg_start_backup(backup_label, fastcheckpoint); PG_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0); { @@ -135,7 +135,7 @@ perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir) * pg_stop_backup() for the user. */ void -SendBaseBackup(const char *backup_label, bool progress) +SendBaseBackup(const char *backup_label, bool progress, bool fastcheckpoint) { DIR *dir; MemoryContext backup_context; @@ -168,7 +168,7 @@ SendBaseBackup(const char *backup_label, bool progress) ereport(ERROR, (errmsg("unable to open directory pg_tblspc: %m"))); - perform_base_backup(backup_label, progress, dir); + perform_base_backup(backup_label, progress, dir, fastcheckpoint); FreeDir(dir); @@ -333,7 +333,16 @@ sendDir(char *path, int basepathlen, bool sizeonly) if (strcmp(pathbuf, "./pg_xlog") == 0) { if (!sizeonly) + { + /* If pg_xlog is a symlink, write it as a directory anyway */ +#ifndef WIN32 + if (S_ISLNK(statbuf.st_mode)) +#else + if (pgwin32_is_junction(pathbuf)) +#endif + statbuf.st_mode = S_IFDIR | S_IRWXU; _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); + } size += 512; /* Size of the header just added */ continue; /* don't recurse into pg_xlog */ } diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 0ef33ddb4f..e4f4c4742f 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -66,11 +66,12 @@ Node *replication_parse_result; %token K_IDENTIFY_SYSTEM %token K_LABEL %token K_PROGRESS +%token K_FAST %token K_START_REPLICATION %type command %type base_backup start_replication identify_system -%type opt_progress +%type opt_progress opt_fast %type opt_label %% @@ -102,15 +103,16 @@ identify_system: ; /* - * BASE_BACKUP [LABEL