diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index f25a983..57b812a 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -33,7 +33,8 @@ export default defineConfig( text: 'Docs', items: [ { text: '7.x', link: '/docs/7x' }, - { text: '6.x', link: '/docs/6x' } + { text: '6.x', link: '/docs/6x' }, + { text: 'Backup & restore', link: '/whpg-backup/' } ] }, { text: 'GitHub', link: 'https://github.com/warehouse-pg/warehouse-pg' }, @@ -45,22 +46,22 @@ export default defineConfig( sidebar: [ { - text: 'WHPG Documentation' + text: 'WHPG documentation' }, { text: "WHPG 7.x", link: "/docs/7x/" , collapsed: false, items: [ - { text: "Release Notes", link: "/docs/7x/release_notes" }, - { text: "Install Guide", link: "/docs/7x/install_guide/" }, - { text: "Admin Guide", link: "/docs/7x/admin_guide/" }, - { text: "Best Practices", link: "/docs/7x/best_practices/" }, - { text: "Utility Guide", link: "/docs/7x/ref_guide/utility_guide/" }, - { text: "Analytics Guide", link: "/docs/7x/admin_guide/analytics/" }, - { text: "Reference Guide", link: "/docs/7x/ref_guide/" }, - { text: "Security Guide", link: "/docs/7x/security_guide/" }, - { text: "Backup & Restore Guide", link: "/docs/7x/admin_guide/backup_restore/" } + { text: "Release notes", link: "/docs/7x/release_notes" }, + { text: "Install guide", link: "/docs/7x/install_guide/" }, + { text: "Admin guide", link: "/docs/7x/admin_guide/" }, + { text: "Best practices", link: "/docs/7x/best_practices/" }, + { text: "Utility guide", link: "/docs/7x/ref_guide/utility_guide/" }, + { text: "Analytics guide", link: "/docs/7x/admin_guide/analytics/" }, + { text: "Reference guide", link: "/docs/7x/ref_guide/" }, + { text: "Security guide", link: "/docs/7x/security_guide/" }, + { text: "Backup & restore guide", link: "/whpg-backup/" } @@ -73,17 +74,33 @@ export default defineConfig( link: "/docs/6x/index.html" , collapsed: true, items: [ - { text: "Release Notes", link: "/docs/6x/release_notes/" }, - { text: "Install Guide", link: "/docs/6x/install_guide/" }, - { text: "Admin Guide", link: "/docs/6x/admin_guide/" }, - { text: "Best Practices", link: "/docs/6x/best_practices/" }, - { text: "Utility Guide", link: "/docs/6x/ref_guide/utility_guide/" }, - { text: "Analytics Guide", link: "/docs/6x/admin_guide/analytics/" }, - { text: "Reference Guide", link: "/docs/6x/ref_guide/" }, - { text: "Security Guide", link: "/docs/6x/security-guide/" }, + { text: "Release notes", link: "/docs/6x/release_notes/" }, + { text: "Install guide", link: "/docs/6x/install_guide/" }, + { text: "Admin guide", link: "/docs/6x/admin_guide/" }, + { text: "Best practices", link: "/docs/6x/best_practices/" }, + { text: "Utility guide", link: "/docs/6x/ref_guide/utility_guide/" }, + { text: "Analytics guide", link: "/docs/6x/admin_guide/analytics/" }, + { text: "Reference guide", link: "/docs/6x/ref_guide/" }, + { text: "Security guide", link: "/docs/6x/security-guide/" }, ], + }, + { + text: "WHPG backup & restore", + link: "/whpg-backup/" , + collapsed: false, + items: [ + { + text: "Release notes", link: "/whpg-backup/release_notes/"}, + { text: "Overview", link: "/whpg-backup/overview/" }, + { text: "Installing", link: "/whpg-backup/installing" }, + { text: "Backing up and restoring", link: "/whpg-backup/using" }, + { text: "Creating incremental backups", link: "/whpg-backup/incremental" }, + { text: "Using the S3 storage plugin", link: "/whpg-backup/s3-plugin" }, + { text: "Reference", + link: "/whpg-backup/reference/" } + ], } ] diff --git a/docs/whpg-backup/images/parallel_backup_restore.png b/docs/whpg-backup/images/parallel_backup_restore.png new file mode 100644 index 0000000..f05a6d1 Binary files /dev/null and b/docs/whpg-backup/images/parallel_backup_restore.png differ diff --git a/docs/whpg-backup/incremental.md b/docs/whpg-backup/incremental.md new file mode 100644 index 0000000..e4a6889 --- /dev/null +++ b/docs/whpg-backup/incremental.md @@ -0,0 +1,99 @@ +--- +title: Creating incremental backups of append-optimized tables + +--- + +Use `gpbackup` and `gprestore` to create and restore incremental backups of append-optimized tables. Full backups contain every object in the database, making a restore, filtered or complete, straightforward, but they cost more in size and duration. Incremental backups reduce that cost when the total amount of changed append-optimized data is small compared to the data that hasn't changed since the last backup. + +`gpbackup` backs up an append-optimized table in an incremental backup only if one of these operations ran against the table after the last full or incremental backup: + +- `ALTER TABLE` +- `DELETE` +- `INSERT` +- `TRUNCATE` +- `UPDATE` +- `DROP` and then re-create the table + +An incremental backup always includes every specified heap table, and backs up an append-optimized table, including a column-oriented table, only if it's changed. An incremental backup set consists of a full backup, plus the incremental backups that capture changes to the database since that full backup. For example, a full backup on Sunday plus three daily incremental backups on Monday, Wednesday, and Friday together form one backup set. Restoring from an incremental backup requires the complete backup set, and every backup in a set must use consistent command-line options so that `gpbackup` can build on it and `gprestore` can restore from it. + +::: info Note +`gpbackup` always backs up a whole table when it includes it in an incremental backup, whether that's because it's a heap table or because an append-optimized table changed. It doesn't back up just the changed rows within a table. For a partitioned append-optimized table, though, `gpbackup` backs up only the leaf partitions that changed, not the entire table, so partitioning large append-optimized tables can substantially reduce the size of your incremental backups. +::: + +## Creating an incremental backup set + +Start an incremental backup set with a full backup, then add incremental backups on top of it using a few required options, plus optional ones for more control. `gpbackup` checks that later backups in the set stay consistent with the ones already there. + +```bash +gpbackup --dbname --incremental --backup-dir --leaf-partition-data +``` + +Include these options when you create an incremental backup: + +- **`--leaf-partition-data`**: Required for every backup in the set, including the full backup that serves as its base. +- **`--incremental`**: Required to create an incremental backup. Not compatible with `--data-only` or `--metadata-only`. +- **`--from-timestamp`** (optional): Use with `--incremental` to specify the timestamp of an existing backup, either an incremental backup or the initial full backup, to build on. The new backup must use the same command-line options as the backup specified. If you don't specify `--from-timestamp`, `gpbackup` looks for a compatible backup using the [backup history database](using.md#understanding-the-backup-history-database). + +When you add an incremental backup to a set, `gpbackup` checks that these options match across the full backup and all incremental backups in the set: + +- **`--dbname`**: The database must be the same. +- **`--backup-dir`**: The full backup and its incremental backups must be in the same location. +- **`--single-data-file`**: Must be either specified or absent for every backup in the set. +- **`--plugin-config`**: If specified, it must be specified for every backup in the set, referencing the same plugin binary. +- **`--include-table-file`, `--include-schema`**, and other [filtering options](using.md#filtering-backups-and-restores): Must be the same across the set. For schema filters, `gpbackup` checks only the schema names, not the objects they contain. +- **`--no-compression`**: If specified, it must be specified for every backup in the set. If the full backup uses compression, the incremental backups must too, though they can use different compression levels. The default compression level is 1. + +::: warning +If you try to add an incremental backup to a set and its command-line options aren't consistent with the rest of the set, the backup operation fails. +::: + +::: warning +Changing the WarehousePG segment configuration invalidates incremental backups. After you add or remove segment instances, create a full backup before creating another incremental backup. +::: + +## Planning your backup strategy + +Decide how many full and incremental backups to retain based on your organization's regulatory requirements for backups. Factor in your Recovery Point Objective (RPO), when you design your backup strategy. Combine `--incremental` backups, effective partitioning of append-optimized/column-oriented (AO/CO) tables, and the `--include` and `--exclude` filters for a flexible and performant backup strategy. + +When you archive incremental backups, archive the complete backup set, including all files created on the coordinator and all segments. + +The `--with-stats` option doesn't need to be consistent across a backup set. However, to restore statistics with the `gprestore --with-stats` option, the backup you restore from must have been taken with `--with-stats`. + +### Example + +This example follows a common strategy, combining a weekly full backup with daily incremental backups. Each backup's file name includes a timestamp, in `YYYYMMDDhhmmss` format, identifying when `gpbackup` created it. + +1. Create the first full backup of the set on Sunday: + + ```bash + gpbackup --dbname ww_sales --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data + ``` + +1. Create an incremental backup based on that full backup each following day: + + ```bash + gpbackup --dbname ww_sales --incremental --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data + ``` + +Continuing this pattern from Monday through Saturday results in a set of seven backups: one full backup and six incremental backups, each identified by its own timestamp. At the end of a week-long set like this, continue in one of two ways: + +- Run another full backup the next Sunday to start a new backup set and retire or archive the previous week's set. +- Aggregate the week into a single incremental backup using `--from-timestamp`, pointing at the previous Sunday's full backup: + + ```bash + gpbackup --dbname ww_sales --incremental --from-timestamp 20250518010000 --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data + ``` + + This new backup captures the same changes as the whole week's incremental backups combined, since it's incremental from the same full backup they were. That makes the week's individual incremental backups redundant, and you can delete them to reclaim storage space, leaving just the full backup and this one aggregated incremental backup to cover the entire week. + +## Restoring incremental backup sets + +To restore to a point in the middle of an incremental backup set, specify that backup's timestamp with `gprestore --timestamp`. For example, to restore to Wednesday's incremental backup from the weekly set described above: + +```bash +gprestore --timestamp 20250521040000 --redirect-db ww_sales_pitr --create-db +``` + +Every earlier backup in the set, the Monday and Tuesday incremental backups and the Sunday full backup, must also be available for this restore to succeed. + +Restore operations can start from any backup in the set, but changes captured in incremental backups later than the one you restore from aren't restored. When restoring from an incremental backup set, `gprestore` restores each append-optimized table from its most recent version in the set. Heap tables are always restored from the latest backup in the set. diff --git a/docs/whpg-backup/index.md b/docs/whpg-backup/index.md new file mode 100644 index 0000000..c4d90bd --- /dev/null +++ b/docs/whpg-backup/index.md @@ -0,0 +1,14 @@ +--- +title: WarehousePG Backup and Restore + +--- + +Use WarehousePG Backup and Restore (`whpg-backup`) to create and restore logical backups of your WarehousePG tables in parallel. By default, backups are written to local disk on the coordinator and segment hosts, though you can consolidate them in a shared directory or write them directly to S3-compatible storage. `whpg-backup` writes the metadata and DDL for a backup to the coordinator host, while each segment writes its own table data to CSV files, so backup and restore performance scales with your cluster. + +## Key capabilities + +- Parallel backup and restore across the coordinator and all segment hosts +- Full database backups, plus [incremental backups](incremental.md) that include every heap table but back up append-optimized tables only if they changed +- [Filtering](using.md#filtering-backups-and-restores) backups and restores by schema, table, or leaf partition +- [Email notifications](using.md#setting-up-email-alerts) when a backup or restore completes +- Backing up directly to [Amazon S3 or S3-compatible storage](s3-plugin.md) diff --git a/docs/whpg-backup/installing.md b/docs/whpg-backup/installing.md new file mode 100644 index 0000000..c728ff4 --- /dev/null +++ b/docs/whpg-backup/installing.md @@ -0,0 +1,72 @@ +--- +title: Installing WarehousePG Backup and Restore + +--- + +`whpg-backup` provides the `gpbackup` and `gprestore` utilities. Build them from source, then install the binaries on every host in your WarehousePG cluster. + +## Prerequisites + +- **Build host:** [Go](https://go.dev/doc/install) 1.23 or later, plus `sqlite3`. Building directly on the WarehousePG coordinator avoids a cross-compile step, since it's already running the same OS as your segment hosts. If you build on a macOS workstation instead, use `make build_linux` to cross-compile for the cluster. +- **Every host in the cluster:** `sqlite3`. `gpbackup` splits work between the coordinator and every segment host, so the `sqlite3` runtime dependency needs to be present wherever `gpbackup`, `gprestore`, or `gpbackup_helper` runs, not just on the build host. + +## Building from source + +1. Clone the repository and change into it: + + ```bash + git clone https://github.com/warehouse-pg/whpg-backup.git + cd whpg-backup + ``` + +1. Build the `gpbackup`, `gprestore`, and `gpbackup_helper` binaries: + + ```bash + make depend + make build + ``` + + The binaries are placed in `$HOME/go/bin`. If you're cross-compiling from a macOS workstation, run `make build_linux` instead. + +:::: tip +Add `$HOME/go/bin` to your `PATH` so you can run the binaries directly from the build host: + +```bash +export PATH=$PATH:$HOME/go/bin +``` +:::: + +## Installing on the cluster + +1. Confirm `sqlite3` is installed on every host in the cluster, including the coordinator, standby coordinator, and all segment hosts. It's installed by default on many platforms, but install it if it's missing. + +1. On the coordinator, create a file `all_hosts` which lists all hosts in the WHPG cluster. For example: + + ```ini + cdw + scdw + sdw1 + sdw2 + sdw3 + ``` + +1. Transfer the binaries to all hosts in the cluster and place them on `$PATH`, for example in `$GPHOME/bin`. Use the `gpsync` utility on WarehousePG 7, or the `gpscp` utility on WarehousePG 6: + + ### WHPG 7 + + ```bash + gpsync -f all_hosts $HOME/go/bin/gpbackup $HOME/go/bin/gprestore $HOME/go/bin/gpbackup_helper =:$GPHOME/bin + ``` + + ### WHPG 6 + + ```bash + gpscp -f all_hosts $HOME/go/bin/gpbackup $HOME/go/bin/gprestore $HOME/go/bin/gpbackup_helper =:$GPHOME/bin + ``` + +1. Verify the installation by checking the `gpbackup` and `gprestore` versions: + + ```bash + gpbackup --version + gprestore --version + ``` diff --git a/docs/whpg-backup/overview/architecture.md b/docs/whpg-backup/overview/architecture.md new file mode 100644 index 0000000..d4d0af2 --- /dev/null +++ b/docs/whpg-backup/overview/architecture.md @@ -0,0 +1,46 @@ +--- +title: Architecture of WarehousePG Backup and Restore + +--- + +WarehousePG Backup and Restore (`whpg-backup`) provides the `gpbackup` and `gprestore` utilities, which create and restore logical backups of WarehousePG tables in parallel. Because a WarehousePG cluster spreads data across many segment hosts, `gpbackup` and `gprestore` split backup and restore work across the cluster, so throughput scales with the number of segments. By default, backup files stay on local disk on the coordinator and segment hosts, though you can point them at a shared directory instead or write them directly to [S3-compatible storage](../s3-plugin.md). + +![Parallel restore using parallel backup files](../images/parallel_backup_restore.png "Parallel restore using parallel backup files") + +## How it works + +`gpbackup` creates a full backup of a single database, splitting the work between the coordinator and every segment host. The coordinator writes the backup's metadata and DDL, while each segment writes its own table data to CSV files, using the `COPY ... ON SEGMENT` command. Files are compressed by default; use `--no-compression` to disable it. Each `gpbackup` task runs in a single transaction and acquires an `ACCESS SHARE` lock on each table it backs up. On completion, `gpbackup` returns the backup's timestamp, which identifies that backup for a later restore. + +The metadata files contain everything `gprestore` needs to reconstruct a full backup set in parallel, restoring the coordinator's schema and database objects alongside each segment's table data. Because table data is stored as CSV, other utilities, such as [`gpload`](../../whpg/7x/ref_guide/utility_guide/reference/gpload.md), can also load it, in the same cluster or a different one. + +`gprestore` restores the database objects from a backup by default. Restoring the WarehousePG cluster's global objects, such as roles and tablespaces, is optional. See [Objects included in a backup](#objects-included-in-a-backup) for the distinction. + +For the full walkthrough with example commands and output, see [Running a full backup](../using.md#running-a-full-backup) and [Restoring a full backup](../using.md#restoring-a-full-backup). + +## Objects included in a backup + +`gpbackup` and `gprestore` back up and restore two categories of objects: + +- **Database objects** for the database you specify with the `--dbname` option. They include: + - **Schema and storage objects:** schemas, tables, views, materialized views (DDL only), sequences, indexes, types, and domains. + - **Programmability objects:** functions, procedural language extensions, operators, operator families, and operator classes, casts, conversions, aggregates, and text search parsers, dictionaries, templates, and configurations. + - **Rules and triggers:** rules, and triggers (definitions only; WarehousePG doesn't support triggers). + - **External tables:** readable and writable external tables (DDL only), and protocols. + - **Metadata and configuration:** comments, owners, extensions, session-level configuration parameter settings, and table statistics (only with the `--with-stats` option). + + ::: info Note + `gpbackup` and `gprestore` never include these schemas in a backup set: `gp_toolkit`, `information_schema`, `pg_aoseg`, `pg_bitmapindex`, `pg_catalog`, `pg_toast*`, and `pg_temp*`. + ::: + +- **Global objects** are also backed up by default. They include: + - Databases + - Database-wide configuration parameter settings + - `GRANT` assignments of roles to databases + - Resource group definitions + - Resource queue definitions + - Roles + - Tablespaces + + ::: info Note + `gprestore` only restores global objects if you include the `--with-globals` option. Conversely, `gpbackup --without-globals` skips the backup of global objects. + ::: diff --git a/docs/whpg-backup/overview/compatibility.md b/docs/whpg-backup/overview/compatibility.md new file mode 100644 index 0000000..439690d --- /dev/null +++ b/docs/whpg-backup/overview/compatibility.md @@ -0,0 +1,11 @@ +--- +title: Compatibility for WarehousePG Backup and Restore + +--- + +The WarehousePG-provided `gpbackup` and `gprestore` utilities are compatible with the following WarehousePG versions: + +- WarehousePG 6.27.1 or later +- WarehousePG 7.2.1-WHPG or later + +`gpbackup` and `gprestore` support RHEL 7, 8, and 9, on `x86_64`. diff --git a/docs/whpg-backup/overview/index.md b/docs/whpg-backup/overview/index.md new file mode 100644 index 0000000..acf8c4c --- /dev/null +++ b/docs/whpg-backup/overview/index.md @@ -0,0 +1,10 @@ +--- +title: Overview of WarehousePG Backup and Restore + +--- + +WarehousePG Backup and Restore (`whpg-backup`) creates and restores WarehousePG backup sets in parallel across the coordinator and all segment hosts. + +- [Architecture](architecture.md): Learn how `whpg-backup` coordinates backups and restores across the coordinator and segment hosts. +- [Compatibility](compatibility.md): Check compatible WarehousePG versions and supported platforms. +- [Known issues](known_issues.md): Review current limitations before you deploy. diff --git a/docs/whpg-backup/overview/known_issues.md b/docs/whpg-backup/overview/known_issues.md new file mode 100644 index 0000000..cf234ea --- /dev/null +++ b/docs/whpg-backup/overview/known_issues.md @@ -0,0 +1,24 @@ +--- +title: Known issues + +--- + +These are the currently known issues and limitations of `gpbackup` and `gprestore`. Where applicable, workarounds are included to help mitigate the impact of these issues. + +- If you create an index on a parent partitioned table, `gpbackup` doesn't back up that same index on child partitioned tables of the parent, as creating the same index on a child would cause an error. However, if you exchange a partition, `gpbackup` doesn't detect that the index on the exchanged partition is inherited from the new parent table. In this case, `gpbackup` backs up conflicting `CREATE INDEX` statements, which causes an error when you restore the backup set. + +- You can execute multiple instances of `gpbackup`, but each execution requires a distinct timestamp. + +- Database object filtering is currently limited to schemas and tables. + +- When backing up a partitioned table where some or all leaf partitions are in different schemas from the root partition, the leaf partition table definitions, including the schemas, are backed up as metadata. This occurs even if the backup operation specifies that schemas containing the leaf partitions should be excluded. To control data being backed up for this type of partitioned table, use the `--leaf-partition-data` option. + +- If you use the `gpbackup --single-data-file` option to combine table backups into a single file per segment, you can't perform a parallel restore operation with `gprestore` (you can't set `--jobs` to a value higher than 1). + +- You can't use `--exclude-table-file` with `--leaf-partition-data`. Although you can specify leaf partition names in a file specified with `--exclude-table-file`, `gpbackup` ignores the partition names. + +- Backing up a database with `gpbackup` while simultaneously running DDL commands might cause `gpbackup` to fail, in order to ensure consistency within the backup set. For example, if a table is dropped after the start of the backup operation, `gpbackup` exits and displays the error `relation does not exist`. + +- `gpbackup` might fail when a table is dropped during a backup operation due to table locking issues. `gpbackup` generates a list of tables to back up and acquires an ACCESS SHARE lock on the tables. If an EXCLUSIVE LOCK is held on a table, `gpbackup` acquires the ACCESS SHARE lock after the existing lock is released. If the table no longer exists when `gpbackup` attempts to acquire a lock on the table, `gpbackup` exits with an error. For tables that might be dropped during a backup, exclude the tables from a backup with a `gpbackup` table filtering option such as `--exclude-table` or `--exclude-schema`. + +- A backup created with `gpbackup` can only be restored to a WarehousePG cluster with the same number of segment instances as the source cluster. If you run `gpexpand` to add segments to the cluster, backups made before starting the expansion can't be restored after the expansion completes. diff --git a/docs/whpg/7x/ref_guide/utility_guide/reference/gpbackup.md b/docs/whpg-backup/reference/gpbackup.md similarity index 50% rename from docs/whpg/7x/ref_guide/utility_guide/reference/gpbackup.md rename to docs/whpg-backup/reference/gpbackup.md index 23d78e2..2c19c9c 100644 --- a/docs/whpg/7x/ref_guide/utility_guide/reference/gpbackup.md +++ b/docs/whpg-backup/reference/gpbackup.md @@ -5,8 +5,6 @@ title: gpbackup Create backup metadata and data files of a WarehousePG cluster compatible with the `gprestore` utility - - ## Synopsis ``` @@ -41,247 +39,256 @@ gpbackup --dbname [--with-stats] [--without-globals] -gpbackup --help +gpbackup --help ``` - `gprestore` [reference](gprestore.md) - - +`gprestore` [reference](gprestore.md) ## Description -The `gpbackup` utility backs up the contents of a database into a collection of metadata files and data files that can be used to restore the database at a later time using gprestore. When you back up a database, you can specify table level and schema level filter options to back up specific tables. For example, you can combine schema level and table level options to back up all the tables in a schema except for a single table. - -By default, `gpbackup` backs up objects in the specified database as well as global WarehousePG system objects. Use `--without-globals` to omit global objects. gprestore does not restore global objects by default; use `--with-global`s to restore them. - -See Objects Included in a Backup or Restore for additional information. - -For materialized views, data is not backed up, only the materialized view definition is backed up. - -gpbackup stores the object metadata files and DDL files for a backup in the WarehousePG coordinator data directory by default. WarehousePG segments use the `COPY ... ON SEGMENT` command to store their data for backed-up tables in compressed CSV data files, located in each segment's data directory. See Understanding Backup Files for additional information. - -You can add the `--backup-dir` option to copy all backup files from the WarehousePG coordinator and segment hosts to an absolute path for later use. Additional options are provided to filter the backup set in order to include or exclude specific tables. - -You can create an incremental backup with the` --incremental` option. Incremental backups are efficient when the total amount of data in append-optimized tables or table partitions that changed is small compared to the data has not changed. See Creating and Using Incremental Backups with `gpbackup` and gprestore for information about incremental backups. +The `gpbackup` utility backs up a database into metadata and data files that `gprestore` can use to restore it later. For background on what gets backed up and how `gpbackup` and `gprestore` work together, see [Architecture](../overview/architecture.md). For a full walkthrough, see [Backing up and restoring your tables](../using.md) and [Creating incremental backups of append-optimized tables](../incremental.md). -With the default `--jobs` option (1 job), each `gpbackup` operation uses a single transaction on the WarehousePG coordinator host. The `COPY ... ON SEGMENT` command performs the backup task in parallel on each segment host. The backup process acquires an `ACCESS SHARE` lock on each table that is backed up. During the table locking process, the database should be in a quiescent state. - -When a back up operation completes, `gpbackup` returns a status code. See Return Codes. - -The `gpbackup` utility cannot be run while gpexpand is initializing new segments. Backups created before the expansion cannot be restored with gprestore after the cluster expansion is completed. - -`gpbackup` can send status email notifications after a back up operation completes. You specify when the utility sends the mail and the email recipients in a configuration file. See Configuring Email Notifications. - -:::info Note -This utility uses secure shell (SSH) connections between systems to perform its tasks. In large WarehousePG deployments, cloud deployments, or deployments with a large number of segments per host, this utility may exceed the host's maximum threshold for unauthenticated connections. Consider updating the `SSH MaxStartups` and` MaxSessions` configuration parameters to increase this threshold. For more information about SSH configuration options, refer to the SSH documentation for your Linux distribution. +::: info Note +- `gpbackup` cannot run while `gpexpand` is initializing new segments. Existing backup sets taken with a different cluster configuration may no longer be compatible after an expansion completes. To restore onto a cluster with a different number of segments, use `gprestore`'s `--resize-cluster` option. +- On completion, `gpbackup` returns a status code. See [Return codes](gprestore.md#return-codes). +- `gpbackup` can send status email notifications after it completes. See [Setting up email alerts](../using.md#setting-up-email-alerts). +- This utility uses SSH connections between hosts. In large or cloud deployments, it may exceed a host's maximum threshold for unauthenticated connections. Consider raising the SSH `MaxStartups` and `MaxSessions` configuration parameters. ::: - - -## Parallel Backup with gpbackup and gprestore - -![Parallel Restore Using Parallel Backup Files](../../../images/parallel_backup_restore.png "Parallel Restore Using Parallel Backup Files") - - - ## Options -`--dbname database_name`
+### --dbname database_name + Required. Specifies the database to back up. -`--backup-dir `
-Optional. Copies all required backup files (metadata files and data files) to the specified directory. You must specify directory as an absolute path (not relative). If you do not supply this option, metadata files are created on the WarehousePG coordinator host in the`$MASTER_DATA_DIRECTORY/backups/YYYYMMDD/YYYYMMDDhhmmss/` directory. Segment hosts create CSV data files in the `/backups/YYYYMMDD/YYYYMMDDhhmmss/` directory. When you specify a custom backup directory, files are copied to these paths in subdirectories of the backup directory. +### --backup-dir directory -When invoked together with the` --metadata-only` and `--no-history` options, a backup may have the same timestamp as another backup, provided that each of the two backups has a distinct `--backup-dir` location. +Optional. Copies all required backup files (metadata files and data files) to the specified directory. You must specify directory as an absolute path (not relative). If you do not supply this option, metadata files are created on the WarehousePG coordinator host in the `$COORDINATOR_DATA_DIRECTORY/backups/YYYYMMDD/YYYYMMDDhhmmss/` directory. Segment hosts create CSV data files in the `/backups/YYYYMMDD/YYYYMMDDhhmmss/` directory. When you specify a custom backup directory, files are copied to these paths in subdirectories of the backup directory. + +When invoked together with the `--metadata-only` and `--no-history` options, a backup may have the same timestamp as another backup, provided that each of the two backups has a distinct `--backup-dir` location. You cannot combine this option with the option `--plugin-config`. -`--compression-level `
-Optional. Specifies the compression level (from 1 to 9) used to compress data files. The default is 1. +### --compression-level level + +Optional. Specifies the compression level (from 1 to 9) used to compress data files. The default is 1. -:::info Note +::: info Note `gpbackup` uses compression by default. ::: -`--compression-type `
+### --compression-type type + Optional. Specifies the compression type (gzip or zstd) used to compress data files. The default is gzip. -:::info Note +::: info Note In order to use the zstd compression type, Zstandard must be installed in a `$PATH` accessible by the `gpadmin` user. ::: -`--copy-queue-size int`
-Optional. Specifies the number of `COPY` commands `gpbackup` should enqueue when backing up using the `--single-data-file` option. This option optimizes backup performance by reducing the amount of time spent initializing `COP`Y commands. If you do not set this option to 2 or greater, `gpbackup` enqueues 1 `COPY` command at a time. +### --copy-queue-size int + +Optional. Specifies the number of `COPY` commands `gpbackup` should enqueue when backing up using the `--single-data-file` option. This option optimizes backup performance by reducing the amount of time spent initializing `COPY` commands. If you don't set this option, `gpbackup` enqueues 1 `COPY` command at a time. If you do set it explicitly, the value must be 2 or greater. -:::info Note -This option must be used with the `--single-data-file` option and cannot be used with the `--jobs` option. +::: info Note +This option must be used with the `--single-data-file` option. ::: -`--data-only`
+### --data-only + Optional. Backs up only the table data into CSV files, but does not backup metadata files needed to recreate the tables and other database objects. -`--debug`
+You cannot combine this option with `--metadata-only`, `--incremental`, or `--leaf-partition-data`. + +### --debug + Optional. Displays verbose debug messages during operation. -`--exclude-schema `
+### --exclude-schema schema_name + Optional. Specifies a database schema to exclude from the backup. You can specify this option multiple times to exclude multiple schemas. You cannot combine this option with the option `--include-schema`, `--include-schema-file`, or a table filtering option such as `--include-table`. -[**See Filtering the Contents of a Backup or Restore for more information.**](../../../admin_guide/backup_restore/index.md#filter) +See [Filtering backups and restores](../using.md#filtering-backups-and-restores) for more information. -**See Requirements and Limitations for limitations when leaf partitions of a partitioned table are in different schemas from the root partition.** +### --exclude-schema-file file_name -`--exclude-schema-file `
Optional. Specifies a text file containing a list of schemas to exclude from the backup. Each line in the text file must define a single schema. The file must not include trailing lines. If a schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. You cannot combine this option with the option `--include-schema` or `--include-schema-file`, or a table filtering option such as `--include-table`. -[**See Filtering the Contents of a Backup or Restore for more information.**](../../../admin_guide/backup_restore/index.md#filter) +See [Filtering backups and restores](../using.md#filtering-backups-and-restores) for more information. -**See Requirements and Limitations for limitations when leaf partitions of a partitioned table are in different schemas from the root partition.** +### --exclude-table schema.table -`--exclude-table `
-Optional. Specifies a table to exclude from the backup. The table must be in the format `.`. If a table or schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. You can specify this option multiple times. You cannot combine this option with the option `--exclude-schema`, `--exclude-schema-file`, or another a table filtering option such as `--include-table`. +Optional. Specifies a table to exclude from the backup. The table must be in the format `.`. If a table or schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. You can specify this option multiple times. You cannot combine this option with the option `--exclude-schema`, `--exclude-schema-file`, or another table filtering option such as `--include-table`. If you specify a leaf partition name, `gpbackup` ignores the partition names. The leaf partition is not excluded. -[**See Filtering the Contents of a Backup or Restore for more information.**](../../../admin_guide/backup_restore/index.md#filter) +See [Filtering backups and restores](../using.md#filtering-backups-and-restores) for more information. + +### --exclude-table-file file_name + +Optional. Specifies a text file containing a list of tables to exclude from the backup. Each line in the text file must define a single table using the format `.`. The file must not include trailing lines. If a table or schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. You cannot combine this option with the option `--exclude-schema`, `--exclude-schema-file`, or another table filtering option such as `--include-table`. -`--exclude-table-file `
-Optional. Specifies a text file containing a list of tables to exclude from the backup. Each line in the text file must define a single table using the format `.`. The file must not include trailing lines. If a table or schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. You cannot combine this option with the option `--exclude-schema`, `--exclude-schema-file`, or another a table filtering option such as `--include-table`. +If you specify leaf partition names in a file that is used with `--exclude-table-file`, `gpbackup` ignores the partition names. The leaf partitions are not excluded. -If you specify leaf partition names in a file that is used with --exclude-table-file, `gpbackup` ignores the partition names. The leaf partitions are not excluded. +See [Filtering backups and restores](../using.md#filtering-backups-and-restores) for more information. -[**See Filtering the Contents of a Backup or Restore for more information.**](../../../admin_guide/backup_restore/index.md#filter) +### --include-schema schema_name -`--include-schema `
-Optional. Specifies a database schema to include in the backup. You can specify this option multiple times to include multiple schemas. If you specify this option, any schemas that are not included in subsequent `--include-schema` options are omitted from the backup set. You cannot combine this option with the options `--exclude-schema`, `--exclude-schema-file`,` --exclude-schema-file`, `--include-table`, or `--include-table-file`. See Filtering the Contents of a Backup or Restore for more information. +Optional. Specifies a database schema to include in the backup. You can specify this option multiple times to include multiple schemas. If you specify this option, any schemas that are not included in subsequent `--include-schema` options are omitted from the backup set. You cannot combine this option with the options `--exclude-schema`, `--exclude-schema-file`, `--include-table`, or `--include-table-file`. -`--include-schema-file `
-Optional. Specifies a text file containing a list of schemas to back up. Each line in the text file must define a single schema. The file must not include trailing lines. If a schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. +See [Filtering backups and restores](../using.md#filtering-backups-and-restores) for more information. -[**See Filtering the Contents of a Backup or Restore for more information.**](../../../admin_guide/backup_restore/index.md#filter) +### --include-schema-file file_name -`--include-table `
-Optional. Specifies a table to include in the backup. The table must be in the format `.`. +Optional. Specifies a text file containing a list of schemas to back up. Each line in the text file must define a single schema. The file must not include trailing lines. If a schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. -You can specify this option multiple times. You cannot combine this option with a schema filtering option such as` --include-schema`, or another table filtering option such as `--exclude-table-file`. +See [Filtering backups and restores](../using.md#filtering-backups-and-restores) for more information. + +### --include-table schema.table + +Optional. Specifies a table to include in the backup. The table must be in the format `.`. You can specify this option multiple times. You cannot combine this option with a schema filtering option such as `--include-schema`, or another table filtering option such as `--exclude-table-file`. You can also specify the qualified name of a sequence, a view, or a materialized view. -If you specify this option, the utility does not automatically back up dependent objects. You must also explicitly specify dependent objects that are Required. For example if you back up a view or a materialized view, you must also back up the tables that the view or materialized view uses. If you back up a table that uses a sequence, you must also back up the sequence. +If you specify this option, the utility does not automatically back up dependent objects. You must also explicitly specify dependent objects that are required. For example, if you back up a view or a materialized view, you must also back up the tables that the view or materialized view uses. If you back up a table that uses a sequence, you must also back up the sequence. + +You can optionally specify a table leaf partition name in place of the table name, to include only specific leaf partitions in a backup with the `--leaf-partition-data` option. When a leaf partition is backed up, the leaf partition data is backed up along with the metadata for the partitioned table. -You can optionally specify a table leaf partition name in place of the table name, to include only specific leaf partitions in a backup with the `--leaf-partition-dat`a option. When a leaf partition is backed up, the leaf partition data is backed up along with the metadata for the partitioned table. +See [Filtering backups and restores](../using.md#filtering-backups-and-restores) for more information. -### See Filtering the Contents of a Backup or Restore for more information. +### --include-table-file file_name -`--include-table-file `
-Optional. Specifies a text file containing a list of tables to include in the backup. Each line in the text file must define a single table using the format `.`. The file must not include trailing lines. For information on specifying special characters in schema and table names, see Schema and Table Names. +Optional. Specifies a text file containing a list of tables to include in the backup. Each line in the text file must define a single table using the format `.`. The file must not include trailing lines. If a table or schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. -Any tables not listed in this file are omitted from the backup set. You cannot combine this option with a schema filtering option such as `--include-schem`a, or another table filtering option such as `--exclude-table-file`. +Any tables not listed in this file are omitted from the backup set. You cannot combine this option with a schema filtering option such as `--include-schema`, or another table filtering option such as `--exclude-table-file`. You can also specify the qualified name of a sequence, a view, or a materialized view. -If you specify this option, the utility does not automatically back up dependent objects. You must also explicitly specify dependent objects that are Required. For example if you back up a view or a materialized view, you must also specify the tables that the view or the materialized view uses. If you specify a table that uses a sequence, you must also specify the sequence. +If you specify this option, the utility does not automatically back up dependent objects. You must also explicitly specify dependent objects that are required. For example, if you back up a view or a materialized view, you must also specify the tables that the view or the materialized view uses. If you specify a table that uses a sequence, you must also specify the sequence. You can optionally specify a table leaf partition name in place of the table name, to include only specific leaf partitions in a backup with the `--leaf-partition-data` option. When a leaf partition is backed up, the leaf partition data is backed up along with the metadata for the partitioned table. -### See Filtering the Contents of a Backup or Restore for more information. +See [Filtering backups and restores](../using.md#filtering-backups-and-restores) for more information. + +### --incremental -`--incremental`
Specify this option to add an incremental backup to an incremental backup set. A backup set is a full backup and one or more incremental backups. The backups in the set must be created with a consistent set of backup options to ensure that the backup set can be used in a restore operation. +You must specify `--leaf-partition-data` with this option. You cannot combine this option with `--data-only` or `--metadata-only`. + By default, `gpbackup` attempts to find the most recent existing backup with a consistent set of options. If the backup is a full backup, the utility creates a backup set. If the backup is an incremental backup, the utility adds the backup to the existing backup set. The incremental backup is added as the latest backup in the backup set. You can specify `--from-timestamp` to override the default behavior. -> `--from-timestamp `
-> Optional. Specifies the timestamp of a backup. The specified backup must have backup options that are consistent with the incremental backup that is >being created. If the specified backup is a full backup, the utility creates a backup set. If the specified backup is an incremental backup, the >utility adds the incremental backup to the existing backup set. -> -> You must specify `--leaf-partition-dat`a with this option. You cannot combine this option with` --data-only` or `--metadata-only`. -> -> A backup is not created and the utility returns an error if the backup cannot add the backup to an existing incremental backup set or cannot use the >backup to create a backup set. +See [Creating incremental backups of append-optimized tables](../incremental.md) for more information. + +#### --from-timestamp backup-timestamp -**For information about creating and using incremental backups, see Creating and Using Incremental Backups with gpbackup and gprestore.** +Optional. Specifies the timestamp of a backup. The specified backup must have backup options that are consistent with the incremental backup that is being created. If the specified backup is a full backup, the utility creates a backup set. If the specified backup is an incremental backup, the utility adds the incremental backup to the existing backup set. + +A backup is not created and the utility returns an error if the backup cannot add the backup to an existing incremental backup set or cannot use the backup to create a backup set. + +### --jobs int -`--jobs `
Optional. Specifies the number of jobs to run in parallel when backing up tables. By default, `gpbackup` uses 1 job (database connection). Increasing this number can improve the speed of backing up data. When running multiple jobs, each job backs up tables in separate transactions. Important: If you specify a value higher than 1, the database should be in a quiescent state while the utility acquires a lock on the tables that are being backed up. If the utility cannot acquire a lock on a table being backed up it will exit. -You cannot use this option in combination with the options `--metadata-only`,` --single-data-file`, or `--plugin-config`. +You cannot use this option in combination with the options `--metadata-only` or `--single-data-file`. -:::info Note -This option must be used with the `--single-data-file` option and cannot be used with the `--jobs` option. -When using the `--jobs` flag, there is a potential deadlock scenario to generate a WARNING message in the log files. During the metadata portion of the backup, the main worker process gathers Access Share locks on all the tables in the backup set. During the data portion of the backup, based on the value of the `--jobs` flag, additional workers are created that attempt to take additional Access Share locks on the tables they back up. - -Between the metadata backup and the data backup, if a third party process (operations like `TRUNCATE`, `DROP`, `ALTER`) attempts to access the same tables and obtain an `Exclusive` lock, the worker thread identifies the potential deadlock and hands off the table backup responsibilities to the main worker (that already has an `Access Share` lock on that particular table). A warning message is logged, similar to: `[WARNING]:-Worker 5 could not acquire AccessShareLock for table public.foo.` +::: info Note +If a third-party process (`TRUNCATE`, `DROP`, `ALTER`) takes an `Exclusive` lock on a table between the metadata and data phases, the worker hands that table off to the main worker instead of failing, and logs a warning like `[WARNING]:-Worker 5 could not acquire AccessShareLock for table public.foo.` ::: -`--leaf-partition-data`
-Optional. For partitioned tables, creates one data file per leaf partition instead of one data file for the entire table (the default). Using this option also enables you to specify individual leaf partitions to include in or exclude from a backup, with the `--include-table, `--include-table-file`, `--exclude-table`, and `--exclude-table-file\` options. +### --leaf-partition-data + +Optional. For partitioned tables, creates one data file per leaf partition instead of one data file for the entire table (the default). Using this option also enables you to specify individual leaf partitions to include in or exclude from a backup, with the `--include-table`, `--include-table-file`, `--exclude-table`, and `--exclude-table-file` options. + +### --metadata-only -`--metadata-only`
Optional. Creates only the metadata files (DDL) needed to recreate the database objects, but does not back up the actual table data. -`--no-compression`
+You cannot combine this option with `--data-only`, `--incremental`, or `--leaf-partition-data`. + +### --no-compression + Optional. Do not compress the table data CSV files. -`--no-history`
+### --no-history + Optional. When invoked, `gpbackup` does not write backup run metadata to the history database. This is useful if you prefer not to have an application-maintained SQLite database on your system. However, if you choose not to back up to the history database you lose access to the features of the gpbackup_manager utility. -`--no-inherits`
+### --no-inherits + Optional. Only works when invoked with either the `--include-table` option or the `--include-table-file` option. When invoked, only the metadata of the table itself is backed up, ignoring any inheritance relationships with other tables that would normally cause those tables to also be included in the backup set. -`--plugin-config `
+### --plugin-config config-file_location + Specify the location of the `gpbackup` plugin configuration file, a YAML-formatted text file. The file contains configuration information for the plugin application that `gpbackup` uses during the backup operation. If you specify the `--plugin-config` option when you back up a database, you must specify this option with configuration information for a corresponding plugin application when you restore the database from the backup. You cannot combine this option with the option `--backup-dir`. -### For information about using storage plugin applications, see Using gpbackup Storage Plugins. +See [Using the S3 storage plugin](../s3-plugin.md) for information about using a storage plugin. + +### --quiet -`--quiet`
Optional. Suppress all non-warning, non-error log messages. -`--single-backup-dir`
-Optional. Store all backup files on a given host in a single directory rather than in distinct per-segment subdirectories under the chosen backup directory. For example, files that were previously created in the `/gpseg0/backups//` or `/gpseg1/backups//` directories will now be created under `/backups//`. You may restore backups taken with this option as normal, but there will be a performance penalty when doing so on some systems. However, this protects you from having to manually reorganize files to the correct segment when you pass the `--resize-cluster` restore option to gprestore. +### --single-backup-dir + +Optional. Store all backup files on a given host in a single directory rather than in distinct per-segment subdirectories under the chosen backup directory. For example, files that were previously created in the `/gpseg0/backups//` or `/gpseg1/backups//` directories will now be created under `/backups//`. You may restore backups taken with this option as normal, but there will be a performance penalty when doing so on some systems. However, this protects you from having to manually reorganize files to the correct segment when you pass the `--resize-cluster` restore option to `gprestore`. + You must use the `--backup-dir` option with this option. -`--single-data-file`
+### --single-data-file + Optional. Create a single data file on each segment host for all tables backed up on that segment. By default, each `gpbackup` creates one compressed CSV file for each table that is backed up on the segment. -:::info Note +::: info Note If you use the `--single-data-file` option to combine table backups into a single file per segment, you cannot set the gprestore option `--jobs` to a value higher than 1 to perform a parallel restore operation. ::: -`--verbose`
+### --verbose + Optional. Print verbose log messages. -`--version`
+### --version + Optional. Print the version number and exit. -`--with-stats`
+### --with-stats + Optional. Include query plan statistics in the backup set. -`--without-globals`
+### --without-globals + Optional. Omit the global WarehousePG system objects during backup. -`--help`
-Displays the online help. +### --help - +Displays the online help. ## Examples -To create the database `test` using the default options: +To take a full backup: ``` -createdb test +gpbackup --dbname ww_sales --compression-type zstd ``` -To create the database `demo` using the WarehousePG coordinator on host `gpcoord`, port `54321`, using the `LATIN1` encoding scheme: +To take a full backup with a consolidated, single data file per segment: ``` -createdb -p 54321 -h gpcoord -E LATIN1 demo +gpbackup --dbname ww_sales --single-data-file --no-compression --backup-dir /tmp/single_file ``` -## See Also +To back up only the `emea_customer` schema: + +``` +gpbackup --dbname ww_sales --include-schema emea_customer +``` -[gprestore](gprestore.md) +To take an incremental backup, which requires `--leaf-partition-data`: + +``` +gpbackup --dbname ww_sales --incremental --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data +``` diff --git a/docs/whpg/7x/ref_guide/utility_guide/reference/gprestore.md b/docs/whpg-backup/reference/gprestore.md similarity index 66% rename from docs/whpg/7x/ref_guide/utility_guide/reference/gprestore.md rename to docs/whpg-backup/reference/gprestore.md index c04f10c..aa273ac 100644 --- a/docs/whpg/7x/ref_guide/utility_guide/reference/gprestore.md +++ b/docs/whpg-backup/reference/gprestore.md @@ -3,9 +3,7 @@ title: gprestore --- -Restore a WarehousePG backup set that was created using the `gpbackup` utility. By default `gprestore` will read the metadata and DDL files located in the Coordinator host data directory, and will load the CSV formatted table data stored locally on each segment hosts. - - +Restore a WarehousePG backup set that was created using the `gpbackup` utility. By default `gprestore` will read the metadata and DDL files located in the Coordinator host data directory, and will load the CSV formatted table data stored locally on each segment hosts. ## Synopsis @@ -16,19 +14,21 @@ gprestore --timestamp [--create-db] [--data-only] [--debug] + [--exclude-extension [--exclude-extension ...]] [--exclude-schema [--exclude-schema ...]] [--exclude-schema-file ] [--exclude-table [--exclude-table ...]] [--exclude-table-file ] [--help] + [--ignore-plugin-config] [--include-schema [--include-schema ...]] [--include-schema-file ] [--include-table [--include-table ...]] [--include-table-file ] [--incremental] [--jobs ] - [--metadata-only] - [--on-error-continue] + [--metadata-only] + [--on-error-continue] [--plugin-config ] [--quiet] [--redirect-db ] @@ -47,99 +47,96 @@ gprestore --timestamp gprestore --help ``` - `gpbackup` [reference](gpbackup.md) - - +`gpbackup` [reference](gpbackup.md) ## Description -To use `gprestore` to restore from a backup set, you must include the `--timestamp` option to specify the exact timestamp value (`YYYYMMDDHHMMSS`) of the backup set to restore. If you specified a custom `--backup-dir` to consolidate the backup files, include the same `--backup-dir` option with `gprestore` to locate the backup files. - -If the backup you specify is an incremental backup, you need a complete set of backup files (a full backup and any required incremental backups). `gprestore` ensures that the complete backup set is available before attempting to restore a backup. - -Important: For incremental backup sets, the backups must be on a single device. For example, a backup set must all be on a Data Domain system. - -When restoring from a backup set, `gprestore` restores to a database with the same name as the name specified when creating the backup set. If the target database exists and a table being restored exists in the database, the restore operation fails. Include the `--create-db` option if the target database does not exist in the cluster. You can optionally restore a backup set to a different database by using the `--redirect-db` option. - -When restoring a backup set that contains data from some leaf partitions of a partitioned tables, the partitioned table is restored along with the data for the leaf partitions. For example, you create a backup with the `gpbackup` option `--include-table-file` and the text file lists some leaf partitions of a partitioned table. Restoring the backup creates the partitioned table and restores the data only for the leaf partitions listed in the file. - -By default, only database objects in the backup set are restored. WarehousePG system objects are automatically included in a `gpbackup` backup set, but these objects are only restored if you include the `--with-globals` option to `gprestore`. - -During a restore operation, automatic updating of table statistics is deactivated for the tables being restored. If you backed up query plan statistics using the `--with-stats` option, you can restore those statistics by providing `--with-stats` to `gprestore`. If you did not use `--with-stats` during a backup, or you want to collect current statistics during the restore operation, you can use the `--run-analyze` option to run `ANALYZE` on the restored tables. - -When a materialized view is restored, the data is not restored. To populate the materialized view with data, use `REFRESH MATERIALIZED VIEW`. The tables that are referenced by the materialized view definition must be available. The `gprestore` log file lists the materialized views that were restored and the `REFRESH MATERIALIZED VIEW` commands that are used to populate the materialized views with data. +To restore from a backup set, run `gprestore` with the `--timestamp` option to specify the exact timestamp (`YYYYMMDDHHMMSS`) of the backup set to restore. If the backup used a custom `--backup-dir`, include the same option here to locate the files. For a full walkthrough, see [Backing up and restoring your tables](../using.md). -Performance of restore operations can be improved by creating multiple parallel connections to restore table data and metadata. By default `gprestore` uses 1 connection, but you can increase this number with the `--jobs` option for large restore operations. - -When a restore operation completes, `gprestore` returns a status code. - -`gprestore` can send status email notifications after a back up operation completes. You specify when the utility sends the mail and the email recipients in a configuration file. - -Note: This utility uses secure shell (SSH) connections between systems to perform its tasks. In large WarehousePG deployments, cloud deployments, or deployments with a large number of segments per host, this utility may exceed the host's maximum threshold for unauthenticated connections. Consider updating the SSH `MaxStartups` and `MaxSessions` configuration parameters to increase this threshold. For more information about SSH configuration options, refer to the SSH documentation for your Linux distribution. - - +::: info Note +- If you're restoring an incremental backup, you need the complete backup set. See [Restoring incremental backup sets](../incremental.md#restoring-incremental-backup-sets). +- If the target database already exists and contains a table being restored, the restore operation fails. Include `--create-db` if the target database doesn't exist, or `--redirect-db` to restore into a different one. +- Leaf partition filtering affects what gets restored. See [Filtering backups by leaf partition](../using.md#filtering-backups-by-leaf-partition). +- On completion, `gprestore` returns a status code. See [Return codes](#return-codes). +- `gprestore` can send status email notifications after it completes. See [Setting up email alerts](../using.md#setting-up-email-alerts). +- This utility uses SSH connections between hosts, with the same considerations as [gpbackup](gpbackup.md#description). +::: ## Options -##### --timestamp YYYYMMDDHHMMSS +### --timestamp YYYYMMDDHHMMSS Required. Specifies the timestamp of the `gpbackup` backup set to restore. By default `gprestore` tries to locate metadata files for the timestamp on the WarehousePG Coordinator host in the $COORDINATOR_DATA_DIRECTORY/backups/YYYYMMDD/YYYYMMDDhhmmss/ directory, and CSV data files in the `/backups/YYYYMMDD/YYYYMMDDhhmmss/` directory of each segment host. -##### --backup-dir directory +### --backup-dir directory Optional. Sources all backup files (metadata files and data files) from the specified directory. You must specify directory as an absolute path (not relative). If you do not supply this option, `gprestore` tries to locate metadata files for the timestamp on the WarehousePG Coordinator host in the $COORDINATOR_DATA_DIRECTORY/backups/YYYYMMDD/YYYYMMDDhhmmss/ directory. CSV data files must be available on each segment in the `/backups/YYYYMMDD/YYYYMMDDhhmmss/` directory. Include this option when you specify a custom backup directory with `gpbackup`. You cannot combine this option with the option `--plugin-config`. -##### --copy-queue-size int +### --copy-queue-size int Optional. Specifies the number of `COPY` commands `gprestore` should enqueue when restoring a backup set. This option optimizes restore performance by reducing the amount of time spent initializing `COPY` commands. If you do not set this option to 2 or greater, `gprestore` enqueues 1 `COPY` command at a time. -##### --create-db +### --create-db Optional. Creates the database before restoring the database object metadata. The database is created by cloning the empty standard system database `template0`. -##### --data-only +You cannot combine this option with `--data-only`. + +### --data-only Optional. Restores table data from a backup created with the `gpbackup` utility, without creating the database tables. This option assumes the tables exist in the target database. To restore data for a specific set of tables from a backup set, you can specify an option to include tables or schemas or exclude tables or schemas. Specify the `--with-stats` option to restore table statistics from the backup. +You cannot combine this option with `--with-globals` or `--create-db`. Global metadata is not backed up in table-filtered or data-only backups. + The backup set must contain the table data to be restored. For example, a backup created with the `gpbackup` option `--metadata-only` does not contain table data. `SEQUENCE` values are updated to match the values taken at the time of the backup. To restore only database tables, without restoring the table data, see the option `--metadata-only`. -##### --debug +### --debug Optional. Displays verbose and debug log messages during a restore operation. -##### --exclude-schema schema_name +### --exclude-extension extension_name + +Optional. Skip restoring the specified extension. You can specify this option multiple times to exclude multiple extensions. Use this option when, for example, the extension's control file isn't installed on the target cluster. + +If the extension owns a schema of the same name in the backup set, `gprestore` also excludes that schema, since the schema exists only as a side effect of creating the extension. You cannot combine this option with `--redirect-schema`. -Optional. Specifies a database schema to exclude from the restore operation. You can specify this option multiple times. You cannot combine this option with the option `--include-schema`, `--include-schema-file`, or a table filtering option such as `--include-table`. +### --exclude-schema schema_name -##### --exclude-schema-file file_name +Optional. Specifies a database schema to exclude from the restore operation. You can specify this option multiple times. You cannot combine this option with the option `--include-schema`, `--include-schema-file`, or a table filtering option such as `--include-table`. + +### --exclude-schema-file file_name Optional. Specifies a text file containing a list of schemas to exclude from the backup. Each line in the text file must define a single schema. The file must not include trailing lines. If a schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. You cannot combine this option with the option `--include-schema` or `--include-schema-file`, or a table filtering option such as `--include-table`. -##### --exclude-table schema.table +### --exclude-table schema.table Optional. Specifies a table to exclude from the restore operation. You can specify this option multiple times. The table must be in the format `.`. If a table or schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. You can specify this option multiple times. If the table is not in the backup set, the restore operation fails. You cannot specify a leaf partition of a partitioned table. You cannot combine this option with the option `--exclude-schema`, `--exclude-schema-file`, or another a table filtering option such as `--include-table`. -##### --exclude-table-file file_name +### --exclude-table-file file_name Optional. Specifies a text file containing a list of tables to exclude from the restore operation. Each line in the text file must define a single table using the format `.`. The file must not include trailing lines. If a table or schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. If a table is not in the backup set, the restore operation fails. You cannot specify a leaf partition of a partitioned table. You cannot combine this option with the option `--exclude-schema`, `--exclude-schema-file`, or another a table filtering option such as `--include-table`. -##### --help +### --help Displays the online help. -##### --include-schema schema_name +### --ignore-plugin-config + +Optional. Restore from the local filesystem instead of the plugin recorded in the backup, for backups where the files are already reachable directly (for example, through a BoostFS mount). You cannot combine this option with `--plugin-config`. + +### --include-schema schema_name Optional. Specifies a database schema to restore. You can specify this option multiple times. If you specify this option, any schemas that you specify must be available in the backup set. Any schemas that are not included in subsequent `--include-schema` options are omitted from the restore operation. @@ -147,7 +144,7 @@ If a schema that you specify for inclusion exists in the database, the utility i You cannot use this option if objects in the backup set have dependencies on multiple schemas. -##### --include-schema-file file_name +### --include-schema-file file_name Optional. Specifies a text file containing a list of schemas to restore. Each line in the text file must define a single schema. The file must not include trailing lines. If a schema name uses any character other than a lowercase letter, number, or an underscore character, then you must include that name in double quotes. @@ -155,7 +152,7 @@ The schemas must exist in the backup set. Any schemas not listed in this file ar You cannot use this option if objects in the backup set have dependencies on multiple schemas. -##### --include-table schema.table +### --include-table schema.table Optional. Specifies a table to restore. The table must be in the format `.`. You can specify this option multiple times. You cannot specify a leaf partition of a partitioned table. @@ -165,7 +162,7 @@ If you specify this option, the utility does not automatically restore dependent You cannot combine this option with a schema filtering option such as `--include-schema`, or another table filtering option such as `--exclude-table-file`. -##### --include-table-file file_name +### --include-table-file file_name Optional. Specifies a text file containing a list of tables to restore. Each line in the text file must define a single table using the format `.`. The file must not include trailing lines. Any tables not listed in this file are omitted from the restore operation. You cannot specify a leaf partition of a partitioned table. @@ -177,9 +174,13 @@ For a materialized view, the data is not restored. To populate the materialized If you use the `--include-table-file` option, `gprestore` does not create roles or set the owner of the tables. The utility restores table indexes and rules. Triggers are also restored but are not supported in WarehousePG. -##### --incremental +### --incremental -Optional. Requires the `--data-only option`. Restores only the table data in the incremental backup specified by the `--timestamp` option. Table data is not restored from previous incremental backups in the backup set. **Warning:** This is a Beta feature and is not supported in a production environment. +Optional. Requires the `--data-only` option. Restores only the table data in the incremental backup specified by the `--timestamp` option. Table data is not restored from previous incremental backups in the backup set. + +::: warning +This is a beta feature and isn't supported in a production environment. +::: An incremental backup contains the following table data that can be restored. @@ -191,27 +192,21 @@ When this option is specified, `gprestore` restores table data by truncating the Before performing the restore operation, `gprestore` ensures that the tables being restored exist. If a table does not exist, `gprestore` returns an error and exits. If the `--on-error-continue` option is specified, `gprestore` logs missing tables and attempts to complete the restore operation. -:::info Warning -When this option is specified, `gpbackup` assumes that no changes have been made to the table definitions of the tables being restored, such as adding or removing columns.--truncate-table +::: warning +When this option is specified, `gprestore` assumes that no changes have been made to the table definitions of the tables being restored, such as adding or removing columns. ::: -Optional. Truncate data from a set of tables before restoring the table data from a backup. This option lets you replace table data with data from a backup. Otherwise, table data might be duplicated. - -You must specify the set of tables with either the option `--include-table` or `--include-table-file`. You must also specify `--data-only` to restore table data without creating the tables. - -You can use this option with the `--redirect-db` option. You cannot use this option with `--redirect-schema`. - -##### --jobs int +### --jobs int Optional. Specifies the number of parallel connections to use when restoring table data and metadata. By default, `gprestore` uses 1 connection. Increasing this number can improve the speed of restoring data. **Note:** If you used the `gpbackup --single-data-file` option to combine table backups into a single file per segment, you cannot set `--jobs` to a value higher than 1 to perform a parallel restore operation. -##### --metadata-only +### --metadata-only Optional. Creates database tables from a backup created with the `gpbackup` utility, but does not restore the table data. This option assumes the tables do not exist in the target database. To create a specific set of tables from a backup set, you can specify an option to include tables or schemas or exclude tables or schemas. Specify the option `--with-globals` to restore the WarehousePG system objects. The backup set must contain the DDL for tables to be restored. For example, a backup created with the `gpbackup` option `--data-only` does not contain the DDL for tables. -##### --on-error-continue +### --on-error-continue Optional. Specify this option to continue the restore operation if an SQL error occurs when creating database metadata (such as tables, roles, or functions) or restoring data. If another type of error occurs, the utility exits. The default is to exit on the first error. @@ -220,81 +215,81 @@ When this option is included, the utility displays an error summary and writes e - Tables with metadata errors - `gprestore___error_tables_metadata` - Tables with data errors - `gprestore___error_tables_data` -##### --plugin-config config-file_location +### --plugin-config config-file_location Specify the location of the `gpbackup` plugin configuration file, a YAML-formatted text file. The file contains configuration information for the plugin application that `gprestore` uses during the restore operation. If you specify the `--plugin-config` option when you back up a database, you must specify this option with configuration information for a corresponding plugin application when you restore the database from the backup. -You cannot combine this option with the option `--backup-dir`. +You cannot combine this option with the option `--backup-dir` or `--ignore-plugin-config`. -##### --quiet +### --quiet Optional. Suppress all non-warning, non-error log messages. -##### --redirect-db database_name +### --redirect-db database_name Optional. Restore to the specified database_name instead of to the database that was backed up. -##### --redirect-schema schema_name +### --redirect-schema schema_name Optional. Restore data in the specified schema instead of the original schemas. The specified schema must already exist. If the data being restored is in multiple schemas, all the data is redirected into the specified schema. This option must be used with an option that includes tables or schemas: `--include-table`, `--include-table-file`, `--include-schema`, or `--include-schema-file`. -You cannot use this option with an option that excludes schemas or tables such as `--exclude-schema` or `--exclude-table`. +You cannot use this option with an option that excludes schemas or tables, such as `--exclude-schema`, `--exclude-table`, or `--exclude-extension`. You can use this option with the `--metadata-only` or `--data-only` options. -##### --report-dir /path/to/report +### --report-dir /path/to/report Optional. The absolute path of the directory to which restore report and error tables will be written. -##### --resize-cluster +### --resize-cluster Optional. Invoke this option to enable restoring data to a cluster that has a different number of segments than the cluster from which the data was backed up. -Warning: This is a Beta feature and is not supported in a production environment. - -:::info Note -In order to enable the `--resize-cluster` feature for `gprestore`, the backup set must have been taking using using `gpbackup` 1.26 or later. +::: info Note +In order to enable the `--resize-cluster` feature for `gprestore`, the backup set must have been taken using `gpbackup` 1.26 or later. ::: -##### --run-analyze +### --run-analyze Optional. Run `ANALYZE` on the tables that are restored. For a partitioned table, `ANALYZE` is run on the root partitioned table. If `--with-stats` was specified for the backup, those statistics are ignored. You cannot use this option with `--with-stats`. If the backup being restored used the `gpbackup` option `--leaf-partition-data`, `gprestore` runs `ANALYZE` only on the individual leaf partitions that are restored, not the root partitioned table. -Depending the tables being restored, running `ANALYZE` on restored tables might increase the duration of the restore operation. +Depending on the tables being restored, running `ANALYZE` on restored tables might increase the duration of the restore operation. -##### --timestamp YYYYMMDDHHMMSS +### --truncate-table -Required. Specifies the timestamp of the `gpbackup` backup set to restore. By default `gprestore` tries to locate metadata files for the timestamp on the WarehousePG Coordinator host in the $COORDINATOR_DATA_DIRECTORY/backups/YYYYMMDD/YYYYMMDDhhmmss/ directory, and CSV data files in the `/backups/YYYYMMDD/YYYYMMDDhhmmss/` directory of each segment host. +Optional. Truncate data from a set of tables before restoring the table data from a backup. This option lets you replace table data with data from a backup. Otherwise, table data might be duplicated. -##### --truncate-table +You must specify at least one of `--include-table`, `--include-table-file`, or `--data-only`. You cannot combine this option with `--metadata-only` or `--incremental`. -Optional. Removes existing data from the tables being restored, to eliminate the chance of duplicate data. +You can use this option with the `--redirect-db` option. You cannot use this option with `--redirect-schema`. -##### --verbose +### --verbose -Optional. Displays verbose log messages during a restore operation.--versionOptional. Print the version number and exit. +Optional. Displays verbose log messages during a restore operation. -##### --version +### --version -Optional. Displays the verison of the gprestore utility. +Optional. Print the version number and exit. -##### --with-globals +### --with-globals Optional. Restores WarehousePG system objects in the backup set, in addition to database objects. -##### --with-stats +You cannot combine this option with `--data-only`. + +### --with-stats Optional. Restore query plan statistics from the backup set. If the backup set was not created with the `--with-stats` option, an error is returned. Restored tables will only have statistics from the backup. You cannot use this option with `--run-analyze`. To collect current statistics for the restored tables during the restore operation, use the `--run-analyze` option. As an alternative, you can run the `ANALYZE` command on the tables after the tables are restored. -## Return Codes +## Return codes One of these codes is returned after `gprestore` completes. @@ -302,62 +297,52 @@ One of these codes is returned after `gprestore` completes. - 1 -- Restore completed with non-fatal errors. See log file for more information. - 2 -- Restore failed with a fatal error. See log file for more information. - - ## Examples -Create the ww_customer database and restore all schemas and tables in the backup set for the indicated timestamp: +Restore all schemas and tables in the backup set for the specified timestamp, creating the target database: ``` -$ dropdb ww_customer -$ gprestore --timestamp 20250515182209 --create-db +gprestore --timestamp 20250515182209 --create-db ``` -Restore the backup set to the "customer" database instead of the "demo" database that was backed up: +Restore the backup set into the `customer` database instead of the database that was backed up: ``` -$ createdb customer -$ gprestore --timestamp 20250515182209 --redirect-db customer +gprestore --timestamp 20250515182209 --redirect-db customer ``` Restore global WarehousePG metadata and query plan statistics in addition to the database objects: ``` -$ gprestore --timestamp 20250515182209 --create-db --with-globals --with-stats +gprestore --timestamp 20250515182209 --create-db --with-globals --with-stats ``` -Restore, using backup files that were created in the /mnt/nfs/wwhpg_backups directory, creating 8 parallel connections: +Restore using backup files from a custom directory, with 8 parallel connections: ``` -$ gprestore --backup-dir /mnt/nfs/whpg_backups/ --timestamp 20250515182209 --create-db --jobs 8 +gprestore --backup-dir /mnt/nfs/whpg_backups/ --timestamp 20250515182209 --create-db --jobs 8 ``` Restore only the `emea_customer` schema included in the backup set: ``` -$ dropdb ww_customer -$ gprestore --include-schema emea_customer --backup-dir /mnt/nfs/whpg_backups/ --timestamp 20250515182209 --create-db +gprestore --include-schema emea_customer --backup-dir /mnt/nfs/whpg_backups/ --timestamp 20250515182209 --create-db ``` -If you restore from an incremental backup set, all the required files in the backup set must be available to `gprestore`. For example, the following timestamp keys specify an incremental backup set. 20250518010000 - is the full backup and the others are incremental backups. +If you restore from an incremental backup set, every earlier backup in the set must be available to `gprestore`. For example, this backup set has a full backup on Sunday and six incremental backups through Saturday: ``` 20250518010000 (full backup, Sunday 1am) 20250519020000 (incremental backup, Monday 2am) 20250520030000 (incremental backup, Tuesday 3am) -20250521040000 (incremental backup, Wedndesday 4am) +20250521040000 (incremental backup, Wednesday 4am) 20250522050000 (incremental backup, Thursday 5am) 20250523060000 (incremental backup, Friday 6am) 20250524070000 (incremental backup, Saturday 7am) ``` -The following `gprestore` command specifies the timestamp `20250521040000` (Wednesday). The incremental backup with the timestamps `20250520030000` (Tuesday), `20250519020000` (Monday) and the full backup `20250518010000` (Sunday) must be available to perform a restore. +Restoring the Wednesday timestamp requires the Monday and Tuesday incremental backups and the Sunday full backup to also be available: ``` -$ gprestore --timestamp 20250521040000 --redirect-db customer_incra --create-db +gprestore --timestamp 20250521040000 --redirect-db customer_incr --create-db ``` - -## See Also - -[gpbackup](gpbackup.md) diff --git a/docs/whpg-backup/reference/index.md b/docs/whpg-backup/reference/index.md new file mode 100644 index 0000000..adb5245 --- /dev/null +++ b/docs/whpg-backup/reference/index.md @@ -0,0 +1,9 @@ +--- +title: WarehousePG Backup and Restore reference + +--- + +Command reference for WarehousePG Backup and Restore. + +- [gpbackup](gpbackup.md) covers the `gpbackup` command and its options. +- [gprestore](gprestore.md) covers the `gprestore` command, its options, and the return codes shared by both utilities. diff --git a/docs/whpg-backup/release_notes/1.31.0_rel_notes.md b/docs/whpg-backup/release_notes/1.31.0_rel_notes.md new file mode 100644 index 0000000..2b38de5 --- /dev/null +++ b/docs/whpg-backup/release_notes/1.31.0_rel_notes.md @@ -0,0 +1,13 @@ +--- +title: WarehousePG Backup and Restore 1.31.0 release notes + +--- + +**Release date:** January 29, 2026 + +WarehousePG Backup and Restore 1.31.0 includes security fixes. + +## Security + +- Mitigated SQL injection risks by using parameterized queries. +- Fixed a security vulnerability. diff --git a/docs/whpg-backup/release_notes/1.32.0_rel_notes.md b/docs/whpg-backup/release_notes/1.32.0_rel_notes.md new file mode 100644 index 0000000..f2f828d --- /dev/null +++ b/docs/whpg-backup/release_notes/1.32.0_rel_notes.md @@ -0,0 +1,16 @@ +--- +title: WarehousePG Backup and Restore 1.32.0 release notes + +--- + +**Release date:** January 29, 2026 + +WarehousePG Backup and Restore 1.32.0 includes a change and an improvement. + +## Changes + +- **Extension-owned access methods:** `gpbackup` no longer includes extension-owned access methods in a backup. + +## Improvements + +- **Help text on empty invocation:** `gprestore` now prints help text when run without any options. diff --git a/docs/whpg-backup/release_notes/1.32.1_rel_notes.md b/docs/whpg-backup/release_notes/1.32.1_rel_notes.md new file mode 100644 index 0000000..dc7e30e --- /dev/null +++ b/docs/whpg-backup/release_notes/1.32.1_rel_notes.md @@ -0,0 +1,22 @@ +--- +title: WarehousePG Backup and Restore 1.32.1 release notes + +--- + +**Release date:** January 29, 2026 + +WarehousePG Backup and Restore 1.32.1 includes a change, an improvement, and bug fixes. + +## Changes + +- **External tables as foreign tables:** `gpbackup` now backs up external tables as foreign tables for WarehousePG 7 and later. + +## Improvements + +- **Batch processing and parallel restore:** Improved batch processing coordination and parallel restore performance. + +## Bug fixes + +- Fixed a bug where `gprestore` reported `ERROR: .() is not a function`. +- Fixed an issue restoring materialized views that use the append-optimized (AO) or append-optimized, column-oriented (AOCS) access method. +- Fixed an issue backing up external partitioned tables when using the `--leaf-partition-data` option. diff --git a/docs/whpg-backup/release_notes/1.33.0_rel_notes.md b/docs/whpg-backup/release_notes/1.33.0_rel_notes.md new file mode 100644 index 0000000..83208bd --- /dev/null +++ b/docs/whpg-backup/release_notes/1.33.0_rel_notes.md @@ -0,0 +1,17 @@ +--- +title: WarehousePG Backup and Restore 1.33.0 release notes + +--- + +**Release date:** April 22, 2026 + +WarehousePG Backup and Restore 1.33.0 includes new features and a bug fix. + +## Features + +- **PGAA table support:** `gpbackup` and `gprestore` now back up and restore databases that mix Postgres Analytics Accelerator (PGAA) tables with regular heap tables, capturing only PGAA metadata rather than the underlying object storage data. +- Added support for `pg_extension_config_dump` in `gpbackup`. + +## Bug fixes + +- Fixed a nil pointer dereference in `DoCleanup` when backup directory creation fails. diff --git a/docs/whpg-backup/release_notes/1.33.1_rel_notes.md b/docs/whpg-backup/release_notes/1.33.1_rel_notes.md new file mode 100644 index 0000000..c9727e2 --- /dev/null +++ b/docs/whpg-backup/release_notes/1.33.1_rel_notes.md @@ -0,0 +1,17 @@ +--- +title: WarehousePG Backup and Restore 1.33.1 release notes + +--- + +**Release date:** May 21, 2026 + +WarehousePG Backup and Restore 1.33.1 includes a new feature and bug fixes. + +## Features + +- **`--ignore-plugin-config` option for `gprestore`:** Lets `gprestore` read plugin-backed backup files directly, without requiring `--plugin-config` or a hand-edited config YAML. + +## Bug fixes + +- Fixed a hang when backing up with `--single-data-file`. `gpbackup` no longer assumes that tables to back up have sorted object IDs (OIDs) in this mode. Previously, if an extension config table had a lower OID than user tables, the backup helper crashed and hung the backup procedure indefinitely. +- Fixed a hang in `TerminateHangingCopySessions` on WarehousePG 6. After the `pgx/v5` upgrade, terminating a hanging `COPY` session on WarehousePG 6 caused the driver to silently retry the `COPY` on a new connection, creating an orphaned session that deadlocked cleanup. `gpbackup` now cancels the backend instead of terminating it, avoiding the retry. diff --git a/docs/whpg-backup/release_notes/index.md b/docs/whpg-backup/release_notes/index.md new file mode 100644 index 0000000..4f03326 --- /dev/null +++ b/docs/whpg-backup/release_notes/index.md @@ -0,0 +1,14 @@ +--- +title: WarehousePG Backup and Restore release notes + +--- + +The WarehousePG Backup and Restore documentation describes the latest version, including minor releases and patches. The release notes provide information on what was new in each release. + +| Version | Release date | +|---------|--------------| +| [1.33.1](1.33.1_rel_notes.md) | 21 May 2026 | +| [1.33.0](1.33.0_rel_notes.md) | 22 Apr 2026 | +| [1.32.1](1.32.1_rel_notes.md) | 29 Jan 2026 | +| [1.32.0](1.32.0_rel_notes.md) | 29 Jan 2026 | +| [1.31.0](1.31.0_rel_notes.md) | 29 Jan 2026 | diff --git a/docs/whpg-backup/s3-plugin.md b/docs/whpg-backup/s3-plugin.md new file mode 100644 index 0000000..6c490e6 --- /dev/null +++ b/docs/whpg-backup/s3-plugin.md @@ -0,0 +1,142 @@ +--- +title: Using the S3 storage plugin + +--- + +Use the S3 storage plugin to write WarehousePG (WHPG) backups directly to Amazon Simple Storage Service (S3) or any S3-compatible storage server, and restore them without storing large dump files on local segment disks. Run the `gpbackup` and `gprestore` commands with the `--plugin-config` option and a YAML configuration file, using the same configuration file for both backup and restore. + +::: tip Associated topics +- [gpbackup & gprestore user guide](index.md) +- [Incremental backups](incremental.md) +- [gpbackup syntax reference](reference/gpbackup.md) +- [gprestore syntax reference](reference/gprestore.md) +::: + +## Installing the plugin + +The S3 plugin is a separate package from `whpg-backup`, built from the [`warehouse-pg/whpg-backup-s3-plugin`](https://github.com/warehouse-pg/whpg-backup-s3-plugin) repository. + +### Prerequisites + +Build the plugin on a host that has Go installed, at the version given by the `go` directive in the repository's `go.mod` file, or later. The WarehousePG coordinator works well as a build host, since `make install` distributes the binary to your cluster's segment hosts directly from there. + +### Building and installing + +1. Clone the repository and change into it: + + ```shell + git clone https://github.com/warehouse-pg/whpg-backup-s3-plugin.git + cd whpg-backup-s3-plugin + ``` + +1. Build the `gpbackup_s3_plugin` binary: + + ```shell + make build + ``` + + The binary is placed in `$GOPATH/bin`, or `$HOME/go/bin` if `GOPATH` isn't set. + +1. With the WarehousePG environment sourced, install the binary on all segment hosts: + + ```shell + make install + ``` + + The plugin executable must reside in the exact same absolute path on every WarehousePG host (coordinator and segments). + +1. Verify that the binary is present and executable on every host, for example using `gpssh` with the `all_hosts` file described in [Installing WarehousePG Backup and Restore](installing.md): + + ```shell + gpssh -f all_hosts -e "test -x \$GPHOME/bin/gpbackup_s3_plugin && echo ok" + ``` + +## Configuring the plugin + +The plugin requires a YAML configuration file located on the coordinator host. The file supports the following options: + +```yaml +executablepath: +options: + region: + endpoint: + aws_access_key_id: + aws_secret_access_key: + bucket: + folder: + encryption: [on|off] + http_proxy: + backup_max_concurrent_requests: [int] + backup_multipart_chunksize: [string] + restore_max_concurrent_requests: [int] + restore_multipart_chunksize: [string] +``` + +Where: + +- `executablepath`: Absolute path to the plugin, for example `$GPHOME/bin/gpbackup_s3_plugin`. +- `region`: AWS region (ignored if `endpoint` is specified). +- `endpoint`: Custom S3-compatible endpoint. +- `aws_access_key_id`: Your AWS/S3 access key. +- `aws_secret_access_key`: Your AWS/S3 secret key. +- `bucket`: The target S3 bucket (must already exist). +- `folder`: The backup sub-location. Created automatically if missing. +- `encryption`: Enables SSL for the S3 connection. Default is `on`. +- `http_proxy`: (Optional) If your WarehousePG segments do not have direct internet access, provide the URL of your corporate proxy server (e.g., http://proxy.example.com:8080). +- `backup_max_concurrent_requests`: (Optional) Number of threads used to upload parts of a file simultaneously. Use this parameter in conjunction with the `gpbackup --jobs` option to increase your overall backup concurrency. +- `backup_multipart_chunksize`: (Optional) The size of each data chunk uploaded to the S3 bucket. Use this parameter along with the `gpbackup --jobs` option and the `backup_max_concurrent_requests` parameter to fine tune your backups. +- `restore_max_concurrent_requests`: (Optional) Number of threads used to download parts of a file during recovery. +- `restore_multipart_chunksize`: (Optional) The size of each data chunk for each individual part of a file during a multipart transfer from S3 to the segments. Use this parameter along with `restore_max_concurrent_requests` to fine tune your restores. + +::: info Note +- The S3 user you configure to run the backups via `aws_access_key_id` must have upload/delete permissions on that bucket for backups and open/download/view permissions for restores. +- All segment hosts must have network access to the S3 endpoint, or the proxy configured via `http_proxy`. +::: + +## Performing a backup + +To perform a backup, use the `gpbackup` command with the `--plugin-config` option and specify the path to your YAML file: + +```shell +gpbackup --dbname --plugin-config //s3-config.yaml +``` + +The plugin organizes backups files using the following directory structure: + +``` +bucket/folder/backups/YYYYMMDD/YYYYMMDDHHMMSS/ +``` + +## Performing a restore + +To restore a backup created with the S3 plugin, you must use the same `--plugin-config` file. You will also need the timestamp of the backup you wish to restore. + +```shell +gprestore --timestamp --plugin-config //s3-config.yaml +``` + +## Example + +1. Create a configuration file named `s3-test-config.yaml`: + + ```yaml + executablepath: $GPHOME/bin/gpbackup_s3_plugin + options: + region: us-west-2 + aws_access_key_id: test-s3-user + aws_secret_access_key: asdf1234asdf + bucket: gpdb-backup + folder: test/backup3 + ``` + +1. On the coordinator, run the backup: + + ```shell + gpbackup --dbname demo --single-data-file --plugin-config /home/gpadmin/s3-test-config.yaml + ``` + +1. Execute the restore (using the timestamp generated by the backup above): + + ``` + gprestore --timestamp 20260318120000 --plugin-config /home/gpadmin/s3-test-config.yaml + ``` diff --git a/docs/whpg-backup/using.md b/docs/whpg-backup/using.md new file mode 100644 index 0000000..c0d18e2 --- /dev/null +++ b/docs/whpg-backup/using.md @@ -0,0 +1,238 @@ +--- +title: Backing up and restoring your tables + +--- + +Use `gpbackup` and `gprestore` to take and restore full backups of a WarehousePG database, filtering what's included and getting notified by email when a run completes. For incremental backups of append-optimized tables, see [Creating incremental backups of append-optimized tables](incremental.md). For architecture background, see [Architecture](overview/architecture.md). For command syntax, see the [gpbackup](reference/gpbackup.md) and [gprestore](reference/gprestore.md) reference pages. + +## Running a full backup + +To take a full backup that includes metadata and user data, run `gpbackup` with the `--dbname` option. This example uses zstandard compression: + +```bash +gpbackup --dbname ww_sales --compression-type zstd +``` + +::: info Note +In order to use the zstd compression type, Zstandard must be installed in a `$PATH` accessible by the `gpadmin` user. +::: + +Specify the gzip compression level (1-9) with `--compression-level`, or disable compression entirely with `--no-compression`. If you don't specify a level, `gpbackup` uses level 1 by default. + +By default, `gpbackup` writes metadata files to the coordinator's backup directory while each segment writes its own table data locally, so the backup set ends up spread across every host in the cluster. To consolidate all backup files into a single directory, include the `--backup-dir` option with an absolute path. You can also add the `--single-data-file` option to write a single data file per segment instead of one file per table. Combining files this way is useful in situations where the overhead of managing many small files is a problem, such as with some third-party storage solutions: + +```bash +gpbackup --dbname ww_sales --single-data-file --no-compression --backup-dir /tmp/single_file +``` + +`gpbackup` reports progress as it runs, including a timestamped log line for each phase of the backup and a progress bar while it locks and backs up tables, ending with `Backup completed successfully`. + +### Inspecting backup files + +A complete backup set includes metadata files, supporting files, and CSV data files, each named with the timestamp of the backup. By default, metadata and supporting files are stored on the WarehousePG coordinator host, in `$COORDINATOR_DATA_DIRECTORY/backups/YYYYMMDD/YYYYMMDDHHMMSS/`, and each segment stores its own table data in the equivalent path in its data directory. If you specify a custom backup directory with `--backup-dir`, `gpbackup` creates this same path as a subdirectory of it. + +::: warning +`gpbackup` creates all metadata files with read-only permissions. Never delete or modify them. Doing so can make the backup set unusable. +::: + +#### Coordinator files + +The backup directory on the coordinator contains the following metadata files: + +- **`gpbackup__metadata.sql`** contains the DDL for the backup's global and database objects, split into what `gprestore` creates before and after the data is restored. See [Objects included in a backup](overview/architecture.md#objects-included-in-a-backup) for the full list. +- **`gpbackup__toc.yaml`** is the table of contents file. It records where each object's DDL is located in the metadata SQL file, along with the table names and OIDs used to locate the corresponding table data in the [segment data files](#segment-files). For example: + + ```yaml + dataentries: + - schema: public + name: customer + oid: 27289 + attributestring: (c_custkey,c_name,c_address,c_nationkey,c_phone,c_acctbal,c_mktsegment,c_comment) + rowscopied: 0 + partitionroot: "" + isreplicated: false + distbyenum: false + ``` + +- **`gpbackup__report`** contains the information used to populate the [notification email](#setting-up-email-alerts), including the command-line options used, the database backed up, the database version, and the backup type. +- **`gpbackup__config.yaml`** contains metadata about the backup task, including the `gpbackup` version, database name, WarehousePG version, and option settings such as `--no-compression`, `--compression-level`, `--metadata-only`, `--data-only`, and `--with-stats`. + +#### Segment files + +By default, each segment creates one compressed CSV file for each table it backs up, using the name format `gpbackup___.gz`, where `` is the segment's content ID, `` is the backup timestamp, and `` is the table's object ID. + +With `--single-data-file`, each segment instead writes all table data to a single file, `gpbackup__`, with a corresponding `gpbackup___toc.yaml`. The coordinator's `gpbackup__toc.yaml` file references each table's `` to locate its data in these segment files. + +### Understanding the backup history database + +Each time `gpbackup` runs, it records details of the operation, such as the timestamp, command-line options, incremental backup details, and status, in a SQLite database at `$COORDINATOR_DATA_DIRECTORY/gpbackup_history.db`. `gpbackup` doesn't back up `gpbackup_history.db` itself, but you can copy it to a secondary location if you want a backup copy. To skip recording a backup in the history database, include the `--no-history` option. + +`gpbackup` uses `gpbackup_history.db` to build the backup and restore plan for an incremental backup set when you run `gpbackup` with `--incremental` and don't specify `--from-timestamp` to indicate the base backup. See [Creating incremental backups](incremental.md) for details. + +## Restoring a full backup + +To restore from a backup set with `gprestore`, specify the exact timestamp `YYYYMMDDHHMMSS` of the backup with the `--timestamp` option. Add the `--create-db` option if the database doesn't already exist in the cluster. If you used a custom `--backup-dir` when creating the backup, include the same option when restoring it: + +```bash +gprestore --timestamp 20250515182209 --backup-dir /tmp/single_file --create-db +``` + +`gprestore` doesn't restore global metadata for the WarehousePG cluster by default. Include the `--with-globals` option if you need it. + +By default, `gprestore` uses one connection to restore table data and metadata. If your backup set is large and wasn't created with `--single-data-file`, you can reduce restore duration by specifying the number of parallel processes with the `--jobs` option, tuned to your available system resources and database size. + +When restoring to an existing database, `gprestore` assumes the public schema exists. When restoring to a new database with `--create-db`, `gprestore` creates the public schema automatically using the `CREATE DATABASE` command, based on the `template0` database that contains the public schema. + +`gprestore` reports its own progress in the same style as `gpbackup`, ending with `Restore completed successfully`. + +## Filtering backups and restores + +Filter backups and restores with schema-level or table-level options to include or exclude individual objects. Without a filter, `gpbackup` backs up all schemas and tables in the specified database. For the full option syntax, see the [gpbackup](reference/gpbackup.md#options) and [gprestore](reference/gprestore.md#options) reference pages. + +### Filtering backups by schema + +Use `--include-schema`, `--include-schema-file`, `--exclude-schema`, or `--exclude-schema-file` to filter by schema. For example: + +```bash +gpbackup --dbname ww_customer --exclude-schema na_customer +``` + +To filter multiple schemas, repeat `--include-schema` or `--exclude-schema` in the same command, for example: + +```bash +gpbackup --dbname ww_customer --include-schema apac_customer --include-schema emea_customer +``` + +Alternatively, use `--include-schema-file` or `--exclude-schema-file` with a text file that lists one schema per line and has no trailing blank lines: + +```bash +gpbackup --dbname ww_customer --include-schema-file /tmp/nw_states.schema +``` + +::: info Note +You can't combine `--include-schema` or `--include-schema-file` with `--exclude-schema` or `--exclude-schema-file`, or with a table filtering option such as `--include-table`. +::: + +### Filtering backups by table + +Filter tables using `--include-table`, `--exclude-table`, `--include-table-file`, and `--exclude-table-file`. These options take the `schema_name.table_name` format on the command line and in a text file. + +```bash +gpbackup --dbname ww_customer --include-table apac_customer.customer_fact +``` + +Enclose table and schema names that contain uppercase letters or spaces in double quotes, for example `na_customer."ZipCodes"` or `"WW_GEO"."time zones"`. + +When you specify `--include-table` or `--include-table-file`, `gpbackup` and `gprestore` don't automatically back up or restore dependent objects. You must list dependent objects in your filter explicitly. For example, if you back up or restore a view or materialized view, also list the tables it depends on. If you back up or restore a table that contains a sequence, also specify the sequence name. + +::: info Note +You can use the individual table-filtering options multiple times, but you can't use `--include-table` and `--exclude-table` in the same command, and you can't combine any table filtering option with a schema filtering option such as `--include-schema`. +::: + +### Filtering backups by leaf partition + +Use the `--leaf-partition-data` option to filter backups to specific leaf partitions, listing their names in the same `schema_name.table_name` format as for a standalone table. This option also changes how `gpbackup` writes data files, creating one file per leaf partition of a partitioned table instead of one file per table on a segment, which is the default. This per-leaf-partition file behavior doesn't apply if you also use `--single-data-file`, since that option writes a single data file per segment regardless of `--leaf-partition-data`. + +For example, given a partitioned table `ww_inventory` with child partitions named `ww_inventory_1_prt_1` through `ww_inventory_1_prt_30`, the file `/tmp/may_week1.include` lists the partitions for the first week of May to include in the backup: + +``` +"WW_GEO".ww_inventory_1_prt_1 +"WW_GEO".ww_inventory_1_prt_2 +"WW_GEO".ww_inventory_1_prt_3 +``` + +Combine `--include-table-file` with `--leaf-partition-data` to create one data file for each leaf partition listed in the file: + +```bash +gpbackup --dbname ww_sales --include-table-file /tmp/may_week1.include --leaf-partition-data +``` + +::: info Note +`--exclude-table-file` and `--leaf-partition-data` aren't compatible. You can specify leaf partition names in a file used with `--exclude-table-file`, but `gpbackup` ignores them. +::: + +### Filtering restores + +Filter what you restore from a backup set using the same schema and table filtering options as `gpbackup`, namely `--include-schema`, `--include-schema-file`, `--exclude-schema`, `--exclude-schema-file`, `--include-table`, `--include-table-file`, `--exclude-table`, and `--exclude-table-file`. These options work the same way as their `gpbackup` counterparts, with these restrictions: + +- The tables you're restoring must not already exist in the target database. +- The schema or table you're restoring must exist in the backup set, or `gprestore` fails. +- With `--include-schema`, `gprestore` can't restore objects that depend on multiple schemas. +- With `--include-table-file`, `gprestore` doesn't create roles or set table ownership, though it restores table indexes and rules. Triggers are also restored, even though WarehousePG doesn't support them. +- The file you specify with `--include-table-file` can't include a leaf partition name, unlike the equivalent `gpbackup` option. If you backed up specific leaf partitions, specify the partitioned table instead to restore their data. + +When you restore a backup set that contains data for only some leaf partitions of a partitioned table, `gprestore` creates the partitioned table and restores data only for the leaf partitions in the backup. + +## Setting up email alerts + +Enable email notifications through `sendmail` to alert you when a `gpbackup` or `gprestore` run completes, by creating a `gp_email_contacts.yaml` file in the home directory of the user running the utilities, or in the same directory as the utilities (`$GPHOME/bin`). If both exist, the one in the home directory takes precedence. If neither is present, or the file isn't configured correctly, the log records that no email was sent. + +The email subject line includes the utility name, timestamp, status, and the name of the WarehousePG coordinator. The email body contains the contents of a report file, which `gpbackup` and `gprestore` each generate when a run completes and place in the WarehousePG coordinator backup directory, named `gpbackup__report` or `gprestore___report`. For example: + +``` +WarehousePG Report + +timestamp key: 20250530234643 +gprestore version: 1.30.5 + +database name: test +command line: gprestore --timestamp 20250530234643 --redirect-db test --create-db + +backup segment count: 8 +restore segment count: 8 +duration: 0:00:42 + +restore status: Success +``` + +Use YAML indentation to define the structure of the `gp_email_contacts.yaml` file, with spaces rather than tabs. + +```yaml +contacts: + gpbackup: + - address: name@domain + status: + success: [true | false] + success_with_errors: [true | false] + failure: [true | false] + gprestore: + - address: name@domain + status: + success: [true | false] + success_with_errors: [true | false] + failure: [true | false] +``` + +The file has these sections: + +- **`contacts`** (required): Contains the `gpbackup` and `gprestore` sections. The file can contain one, or both. +- **`gpbackup`** (optional): Begins the `gpbackup` email section. +- **`address`** (required): At least one email address. You can specify multiple addresses, each with its own `status` section. +- **`status`** (required): Specifies when to send an email, based on the completion status of the operation. Specify at least one of these parameters, each at most once. The default is not to send email notifications. + - **`success`** (optional): Send an email if the operation completes without errors. Defaults to `false`. + - **`success_with_errors`** (optional): Send an email if the operation completes with errors. Defaults to `false`. + - **`failure`** (optional): Send an email if the operation fails. Defaults to `false`. +- **`gprestore`** (optional): Begins the `gprestore` email section, using the same `address` and `status` syntax as `gpbackup`. + +To send email to different addresses depending on whether a backup succeeds or fails, and to a single address for restore operations that succeed or complete with errors, use a configuration like this: + +```yaml +contacts: + gpbackup: + - address: dba_backup_success@whpg.io + status: + success: true + - address: dba_backup_failure@whpg.io + status: + success_with_errors: true + failure: true + gprestore: + - address: dba_restore_status@example.com + status: + success: true + success_with_errors: true +``` + +::: info Note +If a status parameter isn't specified correctly, `gpbackup` and `gprestore` don't issue a warning. For example, a misspelled `success` parameter that's set to `true` doesn't trigger a warning, and no email is sent after a successful operation. Test your email notification configuration to confirm it works as expected. +::: diff --git a/docs/whpg/6x/admin_guide/backup_restore/index.md b/docs/whpg/6x/admin_guide/backup_restore/index.md new file mode 100644 index 0000000..44da618 --- /dev/null +++ b/docs/whpg/6x/admin_guide/backup_restore/index.md @@ -0,0 +1,6 @@ +--- +title: Backing Up and Restoring Databases + +--- + +The `gpbackup` and `gprestore` documentation has moved to the [WarehousePG Backup and Restore](../../../../whpg-backup/index.md) section. diff --git a/docs/whpg/6x/admin_guide/index.md b/docs/whpg/6x/admin_guide/index.md index 87b270b..8610577 100644 --- a/docs/whpg/6x/admin_guide/index.md +++ b/docs/whpg/6x/admin_guide/index.md @@ -14,6 +14,7 @@ navigation: - load - performance - parallel_retrieve_cursor + - backup_restore redirects: - admin_guide - managing_data diff --git a/docs/whpg/6x/admin_guide/managing/backup-main.md b/docs/whpg/6x/admin_guide/managing/backup-main.md index d9041e5..8d47816 100644 --- a/docs/whpg/6x/admin_guide/managing/backup-main.md +++ b/docs/whpg/6x/admin_guide/managing/backup-main.md @@ -11,9 +11,7 @@ WarehousePG supports parallel and non-parallel methods for backing up and restor ## Parallel Backup with gpbackup and gprestore -`gpbackup` and `gprestore` are the recommended WarehousePG backup and restore utilities. `gpbackup` utilizes `ACCESS SHARE` locks at the individual table level, instead of `EXCLUSIVE` locks on the `pg_class` catalog table. This enables you to run DML statements during the backup, such as `CREATE`, `ALTER`, `DROP`, and `TRUNCATE` operations, as long as those operations do not target the current backup set. Backup files created with `gpbackup` are designed to provide future capabilities for restoring individual database objects along with their dependencies, such as functions and required user-defined datatypes. - -`gpbackup`, `gprestore`, and related utilities are provided as a separate download. +`gpbackup` and `gprestore` are the recommended WarehousePG backup and restore utilities, provided as a separate download. For documentation, see [WarehousePG Backup and Restore](../../../../whpg-backup/index.md). diff --git a/docs/whpg/6x/ref_guide/utility_guide/reference/index.md b/docs/whpg/6x/ref_guide/utility_guide/reference/index.md index 7a2ac2b..5cc5757 100644 --- a/docs/whpg/6x/ref_guide/utility_guide/reference/index.md +++ b/docs/whpg/6x/ref_guide/utility_guide/reference/index.md @@ -9,9 +9,9 @@ The command-line utilities provided with WarehousePG. WarehousePG uses the standard PostgreSQL client and server programs and provides additional management utilities for administering a distributed WarehousePG DBMS. -Several utilities are installed when you install the WarehousePG server. These utilities reside in `$GPHOME/bin`. Other utilities must be downloaded and installed separately. These include: +Several utilities are installed when you install the WarehousePG server. These utilities reside in `$GPHOME/bin`. -- The WarehousePG Backup and Restore utilities. +For the `gpbackup` and `gprestore` utilities, see [WarehousePG Backup and Restore](../../../../../whpg-backup/index.md). WarehousePG provides the following utility programs. Superscripts identify those utilities that require separate downloads, as well as those utilities that are also installed with the Client and Loader Tools Packages. (See the Note following the table.) All utilities are installed when you install the WarehousePG server, unless specifically identified by a superscript. diff --git a/docs/whpg/7x/admin_guide/backup_restore/incremental.md b/docs/whpg/7x/admin_guide/backup_restore/incremental.md deleted file mode 100644 index 3d6b168..0000000 --- a/docs/whpg/7x/admin_guide/backup_restore/incremental.md +++ /dev/null @@ -1,360 +0,0 @@ ---- -title: Incremental Backups using gpbackup and gprestore - ---- - -:::tip Associated Topics -- [gpbackup & gprestore user guide](index.md) -- [gpbackup syntax reference](../../ref_guide/utility_guide/reference/gpbackup.md) -- [gprestore syntax reference](../../ref_guide/utility_guide/reference/gprestore.md) -::: - -The [gpbackup](../../ref_guide/utility_guide/reference/gpbackup.md) and [gprestore](../../ref_guide/utility_guide/reference/gprestore.md) utilities support creating -incremental backups of append-optimized tables and restoring from incremental backups. An -incremental backup backs up all specified heap tables and backs up append-optimized tables -(including append-optimized, column-oriented tables) only if the tables have changed. - -For example, if a row of an append-optimized table has changed, the table is backed up. For -partitioned append-optimized tables, only the changed leaf partitions are backed up. - -Incremental backups are efficient when the total amount of data in append-optimized tables or -table partitions that changed is small compared to the data that has not changed since the -last backup. - -An incremental backup backs up an append-optimized table only if one of the following -operations was performed on the table after the last full or incremental backup: - -- `ALTER TABLE` -- `DELETE` -- `INSERT` -- `TRUNCATE` -- `UPDATE` -- `DROP` and then re-create the table - -Restoring data from incremental backups requires a complete incremental backup file set. - - - -## Understanding Incremental Backup Sets - -An incremental backup set includes the following backups: - -- A full backup. This is the full backup that the incremental backups are based on. -- The set of incremental backups that capture the changes to the database from the time - of the full backup. - -For example, you can create a full backup (Sunday) and then create three daily incremental backups(Monday, Wednesday, Friday). -The full backup and all three incremental backups are the backup set. - -:::tip Incremental Backup Set -- Full: Sunday -- Incremental: Monday -- Incremental: Wednesday -- Incremental: Friday -::: - -When you create or add to an incremental backup set, `gpbackup` ensures that the backups in the set are created with a consistent set of backup options to ensure that -the backup set can be used in a restore operation. - -When you create an incremental backup you include these options with the other -`gpbackup` options to create a backup: - -##### `--leaf-partition-data` - -- Required for all backups in the incremental backup set. -- Required when you create a full backup that will be the base backup for an - incremental backup set. - -##### `--incremental` - -- Required when you create an incremental backup. - -:::tip Note -`--data-only` and `--metadata-only` are not compatible with the `--incremental` option -::: - -##### `--from-timestamp` - -- Optional. This option can be used with `--incremental`. The timestamp you specify if of an existing backup. The - timestamp can be either an incremental backup or the initial full backup of the set. - -The backup being created -must be have the same command line options as the with the backup specified with the `--from-timestamp` -option. - -If you do not specify `--from-timestamp`, `gpbackup` attempts to find a compatible backup based on information -in the `gpbackup` history database. - -When you add an incremental backup to a backup set, `gpbackup` ensures that -the full backup and the incremental backups are consistent by checking these -`gpbackup` options: - -##### `--dbname` - The database must be the same. - -##### `--backup-dir` - The directory must be the same. The backup set, the - -full backup and the incremental backups, must be in the same location. - -##### `--single-data-file` - This option must be either specified or absent - -for all backups in the set. - -##### `--plugin-config` - If this option is specified, it must be specified - -for all backups in the backup set. The configuration must reference the same plugin -binary. - -##### `--include-table-file`, `--include-schema`, or any other - -options that filter tables and schemas must be the same. - -**When checking schema filters**, only the schema names are checked, not the objects contained in the -schemas. - -##### `--no-compression` - If this option is specified, it must be specified - -for all backups in the backup set. - -If compression is used on the on the full backup, -compression must be used on the incremental backups. Different compression levels are -allowed for the backups in the backup set. For a backup, the default is compression -level 1. - -:::warning -If you try to add an `--incremental` backup to a backup set, the backup operation fails if the -`gpbackup` command line options options are not consistent. -::: - -:::tip Associated Topics -- [gpbackup & gprestore user guide](index.md) -- [gpbackup syntax reference](../../ref_guide/utility_guide/reference/gpbackup.md) -- [gprestore syntax reference](../../ref_guide/utility_guide/reference/gprestore.md) -::: - - - -## Examples - -The following example shows a backup strategy using weekly Full + daily Incremental backups. - -The following command creates the first full backup of the backup set: - -### Sunday - -``` -$ gpbackup --dbname ww_sales --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data -``` - -Resulting in the example backup set - -`20250518010000 (full backup, Sunday 1am)` - -:::tip -Each backup has a timestamp taken when the backup is created, and is used in the naming of the backup set files. - -`YYYYMMDDhhmmss` is the format. - -For example, a backup on May 18,2025 at 1:00:00am local time, the backup file names contain -`20250527` for the day and `010000` representing 1:00:00am. - -Resulting in a naming convention of `20250518010000` for the backup file set - -When you specify the `--backup-dir` option, the backups are created in the corresponding directory on each WarehousePG host, `/nfsmount/whpg_backups/ww_sales` in this example. -::: - -The next backup will be an incremental backup, based on the full backup above - -### Monday - -``` -$ gpbackup --dbname ww_sales --incremental --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data -``` - -Resulting in a second backup belonging to the backup set. - -``` -20250518010000 (full backup, Sunday 1am) -20250519020000 (incremental backup, Monday 2am) -``` - -Subsequent backups are run on - -- **Tuesday** -- **Wednesday** -- **Thursday** -- **Friday** -- **Saturday** - -### resulting in this complete set. - -``` -20250518010000 (full backup, Sunday 1am) -20250519020000 (incremental backup, Monday 2am) -20250520030000 (incremental backup, Tuesday 3am) -20250521040000 (incremental backup, Wedndesday 4am) -20250522050000 (incremental backup, Thursday 5am) -20250523060000 (incremental backup, Friday 6am) -20250524070000 (incremental backup, Saturday 7am) -``` - -## Full vs Incremental Backup Strategy - -The advantage of Full backups is that they contain every object in the database, allowing for a simple restore (filtered, or complete). One disadvantage may be the size & duration of this backup. - -Incremental backups decrease the time and volume of data backed up, but they will backup `HEAP` tables every time. To mitigate this, using a partitioned `AO/CO` table approach will greatly reduce incremental backups to only those partitions which have seen changes since the previous backup. - -Given the Sunday-Saturday example above, the user has a choice of how to handle the next round of backups. - -## Weekly Full Backup + Incrementals Example - -The user / DBA may choose to run a full backup again on Sunday (no `--incremental` flag) - -``` -$ gpbackup --dbname ww_sales --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data -``` - -Resulting in a new full backup. - -``` -20250518010000 (full backup, Sunday 1am) -20250519020000 (incremental backup, Monday 2am) -20250520030000 (incremental backup, Tuesday 3am) -20250521040000 (incremental backup, Wedndesday 4am) -20250522050000 (incremental backup, Thursday 5am) -20250523060000 (incremental backup, Friday 6am) -20250524070000 (incremental backup, Saturday 7am) -20250525010000 (full backup, Sunday 1am) -``` - -The choice then could be made to retain the prior seven backups, or purge the data to convserve disk space. - -## Aggregated Incrementals Example using --from-timestamp - -An alternative approach would be to take an incremental backup on the next Sunday using `--from-timestamp` which creates a single incremental backup set from the basis of the initial full backup on the previous Sunday. - -For example: - -``` -$ gpbackup --dbname ww_sales --incremental --from-timestamp 20250518010000 --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data -``` - -Resulting in the following backup sets - -``` -20250518010000 (full backup, Sunday 1am) -20250519020000 (incremental backup, Monday 2am) -20250520030000 (incremental backup, Tuesday 3am) -20250521040000 (incremental backup, Wedndesday 4am) -20250522050000 (incremental backup, Thursday 5am) -20250523060000 (incremental backup, Friday 6am) -20250524070000 (incremental backup, Saturday 7am) -20250525010000 (incremental backup, Sunday 1am) -``` - -While this last backup was incremental, it was incremental from the last full backup's timestamp. -This means all the previous incremental backup sets are now redundant, and not required. - -:::tip redundant backup sets -These backup sets can now be deleted to reclaim storage space - -``` -20250519020000 (incremental backup, Monday 2am) -20250520030000 (incremental backup, Tuesday 3am) -20250521040000 (incremental backup, Wedndesday 4am) -20250522050000 (incremental backup, Thursday 5am) -20250523060000 (incremental backup, Friday 6am) -20250524070000 (incremental backup, Saturday 7am) -``` -::: - -Resulting in the following two backups covering the entire week - -``` -20250518010000 (full backup, Sunday 1am) -20250525010000 (incremental backup, Sunday 1am) -``` - - - -## Restoring Incremental Backup Sets - -Based on the previous examples of full and incremental backups - -``` -20250518010000 (full backup, Sunday 1am) -20250519020000 (incremental backup, Monday 2am) -20250520030000 (incremental backup, Tuesday 3am) -20250521040000 (incremental backup, Wedndesday 4am) -20250522050000 (incremental backup, Thursday 5am) -20250523060000 (incremental backup, Friday 6am) -20250524070000 (incremental backup, Saturday 7am) -``` - -The following `gprestore` command specifies the timestamp `20250521040000` (Wednesday) as the restore to point. - -``` -$ gprestore --timestamp 20250521040000 --redirect-db www_sales_pitr --create-db -``` - -The incremental backup with the timestamps - -- `20250520030000` (Tuesday) -- `20250519020000` (Monday) - -And the full backup - -- `20250518010000` (Sunday) - -**must be available to perform a succesful restore.** - - - -## Tips & Notes - -:::tip Choices -- The amount of backups (full or incremental) retained is a choice for each user. Some enterprises may have regulatory requirements to keep `x of days` of backups. - - -- `RPO` (Recovery Point Objective) 1 day, 1 week, 1 month for example, may play a role in how you develop your backup strategy. - -- Using `--incremental` backups, effective partitioning of `AO/CO` tables, and the use of the `--include` and `--exclude` table and schema filters, allow for a flexible and performant approach to your backup strategy. -::: - -To create an incremental backup, or to restore data from an incremental backup set, you -need the complete backup set. When you archive incremental backups, the complete backup -set must be archived. You must archive all the files created on the coordinator and all -segments. - -Each time `gpbackup` runs, the utility adds backup information to the -backup history database `gpbackup_history.db` located in the WarehousePG `$COORDINATORY_DATA_DIRECTORY`. -This sqlite database contains metadata details of the backup set, such as timestamp, flags used, object contents. -To bypass the inserting of this metadata into the history database, the `--no-history` flag can be included in your `gpbakcup` command. - -When using the `--incremental` flag without the `--from-timestamp` option, `gpbackup` will attempt to find the most recent backup with the same command line options, in the `gpbackup_history.db`. If `gpbackup` cannot find a backup with a consistent -set of options, `gpbackup` displays a warning indicating a full backup must be created before an incremental can be created. - -When using the `--from-timestamp` option with `--incremental`, `gpbackup` ensures that the command line options of the current backup match the options of the previously created incremental backup with the specified timestamp. - -The `gpbackup` option `--with-stats` is not required to be the same for all backups in the backup set. However, to perform a restore operation with -the `gprestore` option `--with-stats` to restore statistics, the backup you specify must have must have used the `--with-stats` when -taking the backup. - -Restore operations can be performed from any backup in the backup set. However, changes -captured in incremental backups later than the backup use to restore database data will -not be restored. - -When restoring from an incremental backup set, `gprestore` checks the -backups and restores each append-optimized table from the most recent version of the -append-optimized table in the backup set. - -Heap tables are always restored from the latest backup in the incremental backup set. - -The incremental back up set, a full backup and associated incremental backups, must be on -a single device. For example, the backups in a backup set must all be on the local filesystem, S3 compatible bucket or a NFS mount point accessible my all hosts in the WarehousePG cluster. - -:::warning -Changes to the WarehousePG segment configuration invalidate -incremental backups. After you change the segment configuration (add or remove segment -instances), you must create a full backup before you can create an incremental -backup. -::: diff --git a/docs/whpg/7x/admin_guide/backup_restore/index.md b/docs/whpg/7x/admin_guide/backup_restore/index.md index 4c41a93..44da618 100644 --- a/docs/whpg/7x/admin_guide/backup_restore/index.md +++ b/docs/whpg/7x/admin_guide/backup_restore/index.md @@ -1,973 +1,6 @@ --- -title: Backing up WarehousePG with gpbackup and gprestore +title: Backing Up and Restoring Databases --- -:::tip Associated Topics -- [Incremental backups](incremental.md) -- [gpbackup syntax reference](../../ref_guide/utility_guide/reference/gpbackup.md) -- [gprestore syntax reference](../../ref_guide/utility_guide/reference/gprestore.md) -::: - -[gpbackup](../../ref_guide/utility_guide/reference/gpbackup.md) and [gprestore](../../ref_guide/utility_guide/reference/gprestore.md) - are WarehousePG utilities that create and restore backup sets in parallel for WarehousePG. By default, `gpbackup` stores only the object metadata files and DDL files for a backup in the WarehousePG Coordinator data directory. WarehousePG segments use the `COPY ... ON SEGMENT` command to store their data for backed-up tables in compressed CSV data files, located in each segment’s backups directory. - - - -## Parallel Backup with gpbackup and gprestore - -![Parallel Restore Using Parallel Backup Files](../../images/parallel_backup_restore.png "Parallel Restore Using Parallel Backup Files") - -The backup metadata files contain all of the information that `gprestore` needs to restore a full backup set in parallel. Backup metadata also provides the framework for restoring only individual objects in the data set, along with any dependent objects, in future versions of `gprestore`. (See Understanding Backup Files for more information.) Storing the table data in CSV files also provides opportunities for using other restore utilities, such as gpload, to load the data either in the same cluster or another cluster. By default, one file is created for each table on the segment. You can specify the `--leaf-partition-data` option with `gpbackup` to create one data file per leaf partition of a partitioned table, instead of a single file. This option also enables you to filter backup sets by leaf partitions. - -Each `gpbackup` task uses a single transaction in WarehousePG. During this transaction, metadata is backed up on the Coordinator host, and data for each table on each segment host is written to CSV backup files using COPY ... ON SEGMENT commands in parallel. The backup process acquires an ACCESS SHARE lock on each table that is backed up. - -For additional documenation, visit the [gpbackup](../../ref_guide/utility_guide/reference/gpbackup.md) and [gprestore](../../ref_guide/utility_guide/reference/gprestore.md) cluster utilities reference page. - -- [Objects in a backup set](#objects) - -- [Backup and Restore Workflow](#workflow) - -- [Backup History Database](#db) - -- [Using the --include and --exclude filters](#filter) - -- [Setting Up Email Alerts](#email) - -- [Backup files layout](#files) - -- [Return Codes](#codes) - -- [Supported WHPG versions](#versions) - -- [Limitations](#limitations) - - - -## Objects in a backup set - -Objects Included in a Backup or Restore -The following table lists the objects that are backed up and restored with `gpbackup` and `gprestore`. - -**Database objects** are backed up for the database you specify with the `--dbname` option. - -**Global objects** (WarehousePG Cluster objects) are also backed up by default, but they are restored only if you include the `--with-globals option` when calling `gprestore`. Conversely, the `gpbackup --without-globals` will skip the backup of global objects - -### Objects that are backed up and restored - -| **Database objects** | | -| -------------------------------------------------- | ---------------------------------------------------------------- | -| Aggregates | Readable External Tables 1 | -| Casts | Rules | -| Comments | Schemas 4 | -| Conversions | Sequences | -| Domains | Session-level configuration parameter settings (GUCs) | -| Extensions | Table statistics 2 | -| Functions | Tables | -| Indexes | Text search parsers, dictionaries, templates, and configurations | -| Materialized Views 1 | Triggers 3 | -| Operators, operator families, and operator classes | Types | -| Owners | Views | -| Procedural language extensions | Writable External Tables 1 | -| Protocols | | - -1. DDL Only -2. when `--with-stats` flag is used -3. While WarehousePG does not support triggers, trigger definitions present in the database are backed up and restored -4. The following schemas are not part of the backup set: gp_toolkit, information_schema, pg_aoseg, pg_bitmapindex, pg_catalog, pg_toast*, pg_temp* - -| **Global objects** | \_ | -| ----------------------------------------------------- | -------------------------- | -| Databases | Resource queue definitions | -| Database-wide configuration parameter settings (GUCs) | Roles | -| GRANT assignments of roles to databases | Tablespaces | -| Resource group definitions | | - -When restoring to an existing database, `gprestore` assumes the public schema exists when restoring objects to the public schema. When restoring to a new database (with the --create-db option), `gprestore` creates the public schema automatically when creating a database with the CREATE DATABASE command. The command uses the template0 database that contains the public schema. - - - -## Backup and Restore Workflow - -### Full Backup - -This command will take a full backup (metadata and user data) using zstandard compression - -`$ gpbackup --dbname --compression-type zstd` - -
-Example: - -``` -[gpadmin@whpg_cdw ~]$ gpbackup --dbname ww_sales --compression-type zstd -20250514:20:27:13 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-gpbackup version = 1.30.5 -20250514:20:27:13 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Greenplum Database Version = 6.27.1 build commit:ab5a612bfdc355ad2d601860dfb70a47778c8dd7 -20250514:20:27:13 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Starting backup of database ww_sales -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Backup Timestamp = 20250514202713 -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Backup Database = ww_sales -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Gathering table state information -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Acquiring ACCESS SHARE locks on tables -Locks acquired: 57 / 57 [==============================================================] 100.00% 0s -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Gathering additional table metadata -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Getting partition definitions -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Getting storage information -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Getting child partitions with altered schema -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Metadata will be written to /data/coordinator/gpseg-1/backups/20250514/20250514202713/gpbackup_20250514202713_metadata.sql -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Writing global database metadata -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Global database metadata backup complete -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Writing pre-data metadata -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Pre-data metadata metadata backup complete -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Writing post-data metadata -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Post-data metadata backup complete -20250514:20:27:14 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Writing data to file -Tables backed up: 33 / 33 [============================================================] 100.00% 7s -20250514:20:27:22 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Data backup complete -20250514:20:27:22 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Skipped data backup of 24 external/foreign table(s). -20250514:20:27:22 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-See /home/gpadmin/gpAdminLogs/gpbackup_20250514.log for a complete list of skipped tables. -20250514:20:27:23 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-/home/gpadmin/gp_email_contacts.yaml list found, /data/coordinator/gpseg-1/backups/20250514/20250514202713/gpbackup_20250514202713_report will be sent -20250514:20:28:23 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Beginning cleanup -20250514:20:28:23 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Cleanup complete -20250514:20:28:23 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-Backup completed successfully - -``` - -The above command creates a file that contains global and database-specific metadata on the WarehousePG Coordinator host in the default directory, `$COORDINATOR_DATA_DIRECTORY/backups///`. - -
-For example: - -``` -[gpadmin@whpg_cdw 20250514202713]$ ls -ltr /data/coordinator/gpseg-1/backups/20250514/20250514202713 -total 2112 --r--r--r-- 1 gpadmin gpadmin 2114041 May 14 20:27 gpbackup_20250514202713_metadata.sql --r--r--r-- 1 gpadmin gpadmin 35787 May 14 20:27 gpbackup_20250514202713_toc.yaml --r--r--r-- 1 gpadmin gpadmin 1437 May 14 20:27 gpbackup_20250514202713_config.yaml --r--r--r-- 1 gpadmin gpadmin 1833 May 14 20:27 gpbackup_20250514202713_report -``` - -By default, each segment stores each table’s data for the backup in a separate compressed CSV file in `/backups///`: - -``` -[gpadmin@mdw 20250514202713]$ ls -ltr /data/primary/gpseg0/backups/20250514/20250514202713 -total 125764 --rw------- 1 gpadmin gpadmin 494 May 14 20:27 gpbackup_0_20250514202713_65582.zst --rw------- 1 gpadmin gpadmin 174123 May 14 20:27 gpbackup_0_20250514202713_67854.zst --rw------- 1 gpadmin gpadmin 3057086 May 14 20:27 gpbackup_0_20250514202713_67861.zst --rw------- 1 gpadmin gpadmin 32450223 May 14 20:27 gpbackup_0_20250514202713_70343.zst --rw------- 1 gpadmin gpadmin 1994875 May 14 20:27 gpbackup_0_20250514202713_71070.zst --rw------- 1 gpadmin gpadmin 475745 May 14 20:27 gpbackup_0_20250514202713_71077.zst --rw------- 1 gpadmin gpadmin 3006034 May 14 20:27 gpbackup_0_20250514202713_71084.zst --rw------- 1 gpadmin gpadmin 528895 May 14 20:27 gpbackup_0_20250514202713_71091.zst --rw------- 1 gpadmin gpadmin 8945 May 14 20:27 gpbackup_0_20250514202713_71098.zst --rw------- 1 gpadmin gpadmin 74 May 14 20:27 gpbackup_0_20250514202713_71105.zst --rw------- 1 gpadmin gpadmin 19507368 May 14 20:27 gpbackup_0_20250514202713_71112.zst --rw------- 1 gpadmin gpadmin 596995 May 14 20:27 gpbackup_0_20250514202713_71326.zst --rw------- 1 gpadmin gpadmin 4688 May 14 20:27 gpbackup_0_20250514202713_71333.zst --rw------- 1 gpadmin gpadmin 251 May 14 20:27 gpbackup_0_20250514202713_71340.zst --rw------- 1 gpadmin gpadmin 289 May 14 20:27 gpbackup_0_20250514202713_71347.zst --rw------- 1 gpadmin gpadmin 638 May 14 20:27 gpbackup_0_20250514202713_71354.zst --rw------- 1 gpadmin gpadmin 4729580 May 14 20:27 gpbackup_0_20250514202713_71361.zst --rw------- 1 gpadmin gpadmin 44197780 May 14 20:27 gpbackup_0_20250514202713_71575.zst --rw------- 1 gpadmin gpadmin 198475 May 14 20:27 gpbackup_0_20250514202713_73562.zst --rw------- 1 gpadmin gpadmin 224 May 14 20:27 gpbackup_0_20250514202713_73569.zst --rw------- 1 gpadmin gpadmin 634 May 14 20:27 gpbackup_0_20250514202713_73576.zst --rw------- 1 gpadmin gpadmin 1245122 May 14 20:27 gpbackup_0_20250514202713_73583.zst --rw------- 1 gpadmin gpadmin 16500913 May 14 20:27 gpbackup_0_20250514202713_73716.zst --rw------- 1 gpadmin gpadmin 1455 May 14 20:27 gpbackup_0_20250514202713_74227.zst --rw------- 1 gpadmin gpadmin 13 May 14 20:27 gpbackup_0_20250514202713_74306.zst --rw------- 1 gpadmin gpadmin 13 May 14 20:27 gpbackup_0_20250514202713_74312.zst --rw------- 1 gpadmin gpadmin 302 May 14 20:27 gpbackup_0_20250514202713_74318.zst --rw------- 1 gpadmin gpadmin 278 May 14 20:27 gpbackup_0_20250514202713_74324.zst --rw------- 1 gpadmin gpadmin 353 May 14 20:27 gpbackup_0_20250514202713_74330.zst --rw------- 1 gpadmin gpadmin 13 May 14 20:27 gpbackup_0_20250514202713_74336.zst --rw------- 1 gpadmin gpadmin 13 May 14 20:27 gpbackup_0_20250514202713_74342.zst --rw------- 1 gpadmin gpadmin 22 May 14 20:27 gpbackup_0_20250514202713_74348.zst --rw------- 1 gpadmin gpadmin 441 May 14 20:27 gpbackup_0_20250514202713_74351.zst -``` - -To consolidate all backup files into a single directory, include the `--backup-dir` option with an absolute path to the location you wish to backup to. Optionally, when performing a backup, the `--single-data-file` option may be used in situations where the additional overhead of multiple files might be prohibitive such as third party storage solutions. - -The command below shows both options in use - -``` - -[gpadmin@whpg_cdw ~]$ gpbackup --dbname ww_sales --single-data-file --no-compression --backup-dir /tmp/single_file -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-gpbackup version = 1.30.5 -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Greenplum Database Version = 6.27.1 build commit:ab5a612bfdc355ad2d601860dfb70a47778c8dd7 -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Starting backup of database ww_sales -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Backup Timestamp = 20250515182209 -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Backup Database = ww_sales -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Gathering table state information -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Acquiring ACCESS SHARE locks on tables -Locks acquired: 57 / 57 [==============================================================] 100.00% 0s -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Gathering additional table metadata -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Getting partition definitions -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Getting storage information -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Getting child partitions with altered schema -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Metadata will be written to /tmp/single_file/gpseg-1/backups/20250515/20250515182209/gpbackup_20250515182209_metadata.sql -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Writing global database metadata -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Global database metadata backup complete -20250515:18:22:09 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Writing pre-data metadata -20250515:18:22:10 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Pre-data metadata metadata backup complete -20250515:18:22:10 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Writing post-data metadata -20250515:18:22:10 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Post-data metadata backup complete -20250515:18:22:10 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Writing data to file -Tables backed up: 33 / 33 [============================================================] 100.00% 8s -20250515:18:22:18 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Data backup complete -20250515:18:22:18 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Skipped data backup of 24 external/foreign table(s). -20250515:18:22:18 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-See /home/gpadmin/gpAdminLogs/gpbackup_20250515.log for a complete list of skipped tables. -20250515:18:22:19 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Found neither /usr/local/greenplum-db-6.27.1/bin/gp_email_contacts.yaml nor /home/gpadmin/gp_email_contacts.yaml -20250515:18:22:19 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Email containing gpbackup report /tmp/single_file/gpseg-1/backups/20250515/20250515182209/gpbackup_20250515182209_report will not be sent -20250515:18:22:19 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Beginning cleanup -20250515:18:22:24 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Cleanup complete -20250515:18:22:24 gpbackup:gpadmin:whpg_cdw:032298-[INFO]:-Backup completed successfully -``` - -The contents of the corresponding backup directories show the single data file, and it's associated table of contents files (toc). Both files are required for a successful restore. - -``` -[gpadmin@whpg_cdw ~]$ ls -ltr /tmp/single_file/gp*/backups/*/* - -/tmp/single_file/gpseg0/backups/20250515/20250515182209: -total 407012 --rw-rw-r-- 1 gpadmin gpadmin 416771752 May 15 18:22 gpbackup_0_20250515182209 --r--r--r-- 1 gpadmin gpadmin 1864 May 15 18:22 gpbackup_0_20250515182209_toc.yaml - -/tmp/single_file/gpseg1/backups/20250515/20250515182209: -total 434396 --rw-rw-r-- 1 gpadmin gpadmin 444811433 May 15 18:22 gpbackup_1_20250515182209 --r--r--r-- 1 gpadmin gpadmin 1864 May 15 18:22 gpbackup_1_20250515182209_toc.yaml - -/tmp/single_file/gpseg2/backups/20250515/20250515182209: -total 407260 --rw-rw-r-- 1 gpadmin gpadmin 417025293 May 15 18:22 gpbackup_2_20250515182209 --r--r--r-- 1 gpadmin gpadmin 1864 May 15 18:22 gpbackup_2_20250515182209_toc.yaml - -/tmp/single_file/gpseg-1/backups/20250515/20250515182209: -total 2112 --r--r--r-- 1 gpadmin gpadmin 2114041 May 15 18:22 gpbackup_20250515182209_metadata.sql --r--r--r-- 1 gpadmin gpadmin 35787 May 15 18:22 gpbackup_20250515182209_toc.yaml --r--r--r-- 1 gpadmin gpadmin 1451 May 15 18:22 gpbackup_20250515182209_config.yaml --r--r--r-- 1 gpadmin gpadmin 1872 May 15 18:22 gpbackup_20250515182209_report -``` - -### Full Restore - -To use `gprestore` to restore from a backup set, you must use the `--timestamp` option to specify the exact timestamp value (YYYYMMDDHHMMSS) to restore. Include the `--create-db` option if the database does not exist in the cluster. If you specified a custom --backup-dir to consolidate the backup files, include the same `--backup-dir` option when using `gprestore` to locate the backup files. - -``` -[gpadmin@whpg_cdw ~]$ psql -c "DROP DATABASE ww_sales"; -DROP DATABASE -[gpadmin@whpg_cdw ~]$ -[gpadmin@whpg_cdw ~]$ gprestore --timestamp 20250515182209 --backup-dir /tmp/single_file --create-db -20250515:18:35:09 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Restore Key = 20250515182209 -20250515:18:35:09 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-gpbackup version = 1.30.5 -20250515:18:35:09 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-gprestore version = 1.30.5 -20250515:18:35:09 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Greenplum Database Version = 6.27.1 build commit:ab5a612bfdc355ad2d601860dfb70a47778c8dd7 -20250515:18:35:09 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Creating database -20250515:18:35:10 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Database creation complete for: ww_sales -20250515:18:35:10 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Restoring pre-data metadata -Pre-data objects restored: 153 / 153 [=================================================] 100.00% 3s -20250515:18:35:14 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Pre-data metadata restore complete -Tables restored: 33 / 33 [============================================================] 100.00% 38s -20250515:18:35:52 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Data restore complete -20250515:18:35:52 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Restoring post-data metadata -20250515:18:35:52 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Post-data metadata restore complete -20250515:18:35:52 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Found neither /usr/local/greenplum-db-6.27.1/bin/gp_email_contacts.yaml nor /home/gpadmin/gp_email_contacts.yaml -20250515:18:35:52 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Email containing gprestore report /tmp/single_file/gpseg-1/backups/20250515/20250515182209/gprestore_20250515182209_20250515183509_report will not be sent -20250515:18:35:52 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Beginning cleanup -20250515:18:36:00 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Cleanup complete -20250515:18:36:00 gprestore:gpadmin:whpg_cdw:032941-[INFO]:-Restore completed successfully -``` - -`gprestore` does not attempt to restore global metadata for the WarehousePG Cluster by default. If this is required, include the `--with-globals` option. - -By default, `gprestore` uses 1 connection to restore table data and metadata. If you have a large backup set which **did not use** the `--single-data-file` option, restore duration can be decreased by specifiying the number of parallel processes using the `--jobs` option. Depending on system resourceds and database size, this number can be tuned for each specific environment. - -``` -[gpadmin@whpg_cdw ~]$ psql -c "DROP DATABASE ww_sales"; -DROP DATABASE -[gpadmin@whpg_cdw ~]$ -[gpadmin@whpg_cdw ~]$ gprestore --timestamp 20250514202713 --create-db --jobs 4 -20250515:18:40:40 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Restore Key = 20250514202713 -20250515:18:40:40 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-gpbackup version = 1.30.5 -20250515:18:40:40 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-gprestore version = 1.30.5 -20250515:18:40:40 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Greenplum Database Version = 6.27.1 build commit:ab5a612bfdc355ad2d601860dfb70a47778c8dd7 -20250515:18:40:40 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Creating database -20250515:18:40:41 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Database creation complete for: ww_sales -20250515:18:40:41 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Restoring pre-data metadata -Pre-data objects restored: 153 / 153 [=================================================] 100.00% 2s -20250515:18:40:43 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Pre-data metadata restore complete -Tables restored: 33 / 33 [============================================================] 100.00% 16s -20250515:18:41:00 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Data restore complete -20250515:18:41:00 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Restoring post-data metadata -20250515:18:41:00 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Post-data metadata restore complete -20250515:18:41:00 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Found neither /usr/local/greenplum-db-6.27.1/bin/gp_email_contacts.yaml nor /home/gpadmin/gp_email_contacts.yaml -20250515:18:41:00 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Email containing gprestore report /data/coordinator/gpseg-1/backups/20250514/20250514202713/gprestore_20250514202713_20250515184040_report will not be sent -20250515:18:41:00 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Beginning cleanup -20250515:18:41:00 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Cleanup complete -20250515:18:41:00 gprestore:gpadmin:whpg_cdw:033527-[INFO]:-Restore completed successfully -[gpadmin@whpg_cdw ~]$ -``` - -### Report Files - -When performing a backup or restore operation, `gpbackup` and `gprestore` generate a report file. When email notification is configured, the email sent contains the contents of the report file. For information about email notification, see Configuring Email Notifications. - -The report file is placed in the WarehousePG Coordinator backup directory. The report file name contains the timestamp of the operation. These are the formats of the `gpbackup` and `gprestore` report file names. - -- `gpbackup__report` -- `gprestore___report` - -For these example report file names, `20250518010000` is the timestamp of the backup and `20250530234643` is the timestamp of the restore operation. - -**`gpbackup_20250518010000_report`** - -**`gprestore_20250518010000_20180213115426_report`** - -This backup directory on a WarehousePG Coordinator host contains both a `gpbackup` and `gprestore` report file. - -``` -[gpadmin@cdw 20250530234643]$ ls -ltr -total 2124 --r--r--r-- 1 gpadmin gpadmin 2120143 May 30 23:46 gpbackup_20250530234643_metadata.sql --r--r--r-- 1 gpadmin gpadmin 38064 May 30 23:46 gpbackup_20250530234643_toc.yaml --r--r--r-- 1 gpadmin gpadmin 1499 May 30 23:46 gpbackup_20250530234643_config.yaml --r--r--r-- 1 gpadmin gpadmin 1837 May 30 23:46 gpbackup_20250530234643_report --r--r--r-- 1 gpadmin gpadmin 539 May 31 00:05 gprestore_20250530234643_20250531000515_report - -``` - -The contents of the report files are similar. This is an example of the contents of a `gprestore` report file. - -``` -WarehosuePG Report - -timestamp key: 20250530234643 -gpdb version: 6.27.1 build commit:ab5a612bfdc355ad2d601860dfb70a47778c8dd7 -gprestore version: 1.30.5 - -database name: test -command line: gprestore --timestamp 20250530234643 --redirect-db test --create-db - -backup segment count: 8 -restore segment count: 8 -start time: Sat May 31 2025 00:05:15 -end time: Sat May 31 2025 00:05:57 -duration: 0:00:42 - -restore status: Success -``` - - - -## Backup History Database - -`gpbackup` stores details of each backup operation in a SQLite database found in `$COORDINATOR_DATA_DIRECTORY/gpbackup_history.db`. Details such as timestamps, command line options, incremental backup details and status are stored in this database. `gpbackup_history.db` is not backed up my `gpbackup`, but can be copied to a secondary location if a backup copy is desired. - -`gpbackup` uses the metadata in `gpbackup_history.db` to create the backup/restore plan for an incremental backup sets when you run `gpbackup` with the `--incremental` option and do not specify the `--from-timesamp` option to indicate the backup that you want to use as the base backup of the incremental backup set. For information about incremental backups, refer to [Incremental Backups with `gpbackup` and `gprestore`](incremental.md). - - - -## Filtering the Contents of a Backup or Restore - -`gpbackup` backs up all schemas and tables in the specified database, unless you exclude or include individual schema or table objects with schema level or table level filter options. - -The schema level options are - -- `--include-schema` -- `--include-schema-file` -- `--exclude-schema` -- `--exclude-schema-file` - command-line options to `gpbackup`. For example, if the `ww_customer` database contains three schemas, `na_customer`, `apac_customer`, `emea_customer`, the following commands back up only the `apac_customer` and `emea_customer` schemas: - -``` -$ gpbackup --dbname ww_customer --exclude-schema na_customer -``` - -Multiple `--include-schema` options may be used in a `gpbackup` or multiple `--exclude-schema` options to filter 2 or many schemas - -``` -$ gpbackup --dbname ww_customer --include-schema apac_customer --include-schema emea_customer -``` - -An alternative to multiple command line options is to use `--include-schema-file` or `--exclude-schema-file`. Each line in the file defines a single schema, and the file cannot contain trailing lines. - -`/tmp/nw_states.schema` contents - -``` -[gpadmin@whpg_cdw tmp]$ cat nw_states.schema -Washington -Oregon -Idaho -Montana -Wyoming -[gpadmin@whpg_cdw tmp]$ -``` - -Example of the command: - -``` -gpbackup --dbname ww_customer --include-schema-file /tmp/nw_states.schema -``` - -Filtering included or excluded tables is similar process to schemas. - -- `--include-table` -- `--exclude-table` -- `--include-table-file` -- `--exclude-table-file ` - -Options use the `schema-name.table-name` format as input on the command line, and in the text file. The individual table filtering options can be used multiple times, but `--include-table` and `--exclude-table` cannot both be used in the same command. - -You can create a list of qualified table names in a text file. Each line in the file defines a single `schema-name.table-name`, and the file cannot contain trailing lines. - -``` -gpbackup --dbname ww_customer --include-table apac_customer.customer_fact -``` - -or - -``` -gpbackup --dbname ww_customer --exclude-table-file /tmp/exclude_tables.file -``` - -where `/tmp/include_tables.file` contents - -``` -[gpadmin@whpg_cdw tmp]$ cat exclude_tables.file -emea_customer.customer_fact -na_customer.customer_fact -[gpadmin@whpg_cdw tmp]$ -``` - -Table and schema names containing upper case letters or spaces must be enclosed with double quotes. - -``` -[gpadmin@whpg_cdw tmp]$ cat exclude_tables_2.file -emea_customer.customer_fact -na_customer.customer_fact -na_customer."ZipCodes" -"WW_GEO".holidays -"WW_GEO"."time zones" -[gpadmin@whpg_cdw tmp]$ -``` - -When `--include-table` or `--include-table-file` is specified on the command line, dependent objects are not automatically backed up or restored. Depenedent objects must be specified in your filter string or file. For example, if you back up or restore a view or materialized view, you must also list the table(s) the view or the materialized view require. If you backup or restore a table containing a sequence, you must also specify the sequence name. - -### Filtering by Leaf Partition - -By default, `gpbackup` creates one file for each table on a data segment. When the `--leaf-partition-data` option is used, `gpbackup` will create one data file per leaf partition of a partitioned table, instead of a single file data file for the root table. You can also filter backups to specific leaf partitions by including the names of the leaf partitions names as you would a stand along table, using the `schema_name.table_name` format. - -``` -gpadmin=# \d+ ww_inventory - Table "WW_GEO.ww_inventory" - Column | Type | Modifiers | Storage | Stats target | Description ------------------------+-------------------+-----------+----------+--------------+------------- - prod_id | integer | | plain | | - prod_sku | character varying | | extended | | - prod_warehouse | character varying | | extended | | - prod_qty_on_hand | integer | | plain | | - prod_date_inventoried | date | | plain | | -Child tables: ww_inventory_1_prt_1, - ww_inventory_1_prt_10, - ww_inventory_1_prt_11, - ww_inventory_1_prt_12, - ww_inventory_1_prt_13, - ww_inventory_1_prt_14, - ww_inventory_1_prt_15, - ww_inventory_1_prt_16, - ww_inventory_1_prt_17, - ww_inventory_1_prt_18, - ww_inventory_1_prt_19, - ww_inventory_1_prt_2, - ww_inventory_1_prt_20, - ww_inventory_1_prt_21, - ww_inventory_1_prt_22, - ww_inventory_1_prt_23, - ww_inventory_1_prt_24, - ww_inventory_1_prt_25, - ww_inventory_1_prt_26, - ww_inventory_1_prt_27, - ww_inventory_1_prt_28, - ww_inventory_1_prt_29, - ww_inventory_1_prt_3, - ww_inventory_1_prt_30, - ww_inventory_1_prt_4, - ww_inventory_1_prt_5, - ww_inventory_1_prt_6, - ww_inventory_1_prt_7, - ww_inventory_1_prt_8, - ww_inventory_1_prt_9 -Distributed by: (prod_id) -Partition by: (prod_date_inventoried) -``` - -The following file `/tmp/may_week1.include` contains the partitions for the first week of May which will be included in our `gpbackup` - -``` -[gpadmin@whpg_cdw ~]$ cat /tmp/may_week1.include -"WW_GEO".ww_inventory_1_prt_1 -"WW_GEO".ww_inventory_1_prt_2 -"WW_GEO".ww_inventory_1_prt_3 -"WW_GEO".ww_inventory_1_prt_4 -"WW_GEO".ww_inventory_1_prt_5 -"WW_GEO".ww_inventory_1_prt_6 -"WW_GEO".ww_inventory_1_prt_7 -``` - -Using the `--include-table-file` and `--leaf-partition-data` options, `gpbackup` will create one data file for each leaf partition specified in the input file - -``` -$ gpbackup --dbname ww_sales --include-table-file /tmp/may_week1.include --leaf-partition-data -``` - -The `--exclude-table-file` option and `--leaf-partition-data` are not compatible. Although you can specify leaf partition names in a file specified with `--exclude-table-file`, `gpbackup` ignores the partition names. - -### Filtering with `gprestore` - -After creating a backup set with `gpbackup`, you can filter the schemas and tables that you want to restore from the backup set using the `gprestore --include-schema` and `--include-table-file` options. These options work in the same way as their `gpbackup` counterparts, but have the following restrictions: - -The tables that you attempt to restore must not already exist in the database. - -If you attempt to restore a schema or table that does not exist in the backup set, the `gprestore` does not run. - -If you use the `--include-schema` option, `gprestore` cannot restore objects that have dependencies on multiple schemas. - -If you use the `--include-table-file` option, `gprestore` does not create roles or set the owner of the tables. The utility restores table indexes and rules. Triggers are also restored but are not supported in WarehousePG. - -The file that you specify with `--include-table-file` cannot include a leaf partition name, as it can when you specify this option with `gpbackup`. If you specified leaf partitions in the backup set, specify the partitioned table to restore the leaf partition data. - -When restoring a backup set that contains data from some leaf partitions of a partitioned table, the partitioned table is restored along with the data for the leaf partitions. For example, you create a backup with the `gpbackup` option `--include-table-file` and the text file lists some leaf partitions of a partitioned table. Restoring the backup creates the partitioned table and restores the data only for the leaf partitions listed in the file. - - - -## Setting Up Email Alerts - -`gpbackup` and `gprestore` can send email notifications via `sendmail` after a backup or restore run completes. - -To have `gpbackup` or `gprestore` send out status email notifications, a `gp_email_contacts.yaml` file must exist in either home directory of the user running `gpbackup` or `gprestore` or the same directory as the utilities ($GPHOME/bin). - -If the `gp_email_contacts.yaml` file is not present, the gpbackup/gprestore log file will have a message similar to: - -``` -20250530:23:46:52 gpbackup:gpadmin:cdw:022290-[INFO]:-Data backup complete -20250530:23:46:52 gpbackup:gpadmin:cdw:022290-[INFO]:-Skipped data backup of 24 external/foreign table(s). -20250530:23:46:52 gpbackup:gpadmin:cdw:022290-[INFO]:-See /home/gpadmin/gpAdminLogs/gpbackup_20250530.log for a complete list of skipped tables. -20250530:23:46:53 gpbackup:gpadmin:cdw:022290-[INFO]:-Found neither /usr/local/greenplum-db-6.27.1/bin/gp_email_contacts.yaml nor /home/gpadmin/gp_email_contacts.yaml -20250530:23:46:53 gpbackup:gpadmin:cdw:022290-[INFO]:-Email containing gpbackup report /data/coordinator/gpseg-1/backups/20250530/20250530234643/gpbackup_20250530234643_report will not be sent -20250530:23:46:53 gpbackup:gpadmin:cdw:022290-[INFO]:-Beginning cleanup -20250530:23:46:53 gpbackup:gpadmin:cdw:022290-[INFO]:-Cleanup complete -20250530:23:46:53 gpbackup:gpadmin:cdw:022290-[INFO]:-Backup completed successfully -``` - -If the `gp_email_contacts.yaml` file is present, can configured properly, a similar message will be printed to the logs: - -```` -20250514:20:27:23 gpbackup:gpadmin:whpg_cdw:027069-[INFO]:-/home/gpadmin/gp_email_contacts.yaml list found, /data/coordinator/gpseg-1/backups/20250514/20250514202713/gpbackup_20250514202713_report will be sent``` -```` - -The `$HOME/gp_email_contacts.yaml` file will override any configurations found in the `$GPHOME/bin/gp_email_contacts.yaml` - -The email subject line includes the utility name, timestamp, status, and the name of the WarehousePG Coordinator. This is an example subject line for a `gpbackup` email. - -``` -gpbackup 20250514202713 on whpg_cdw completed: Success -``` - -or - -``` -gprestore 20250515182209 on whpg_cdw completed: Failure -``` - -The email contains summary information about the operation including options, duration, and number of objects backed up or restored. For information about the contents of a notification email, see Report Files. - -### gp_email_contacts.yaml file format - -The `gpbackup` and `gprestore` email notification YAML file gp_email_contacts.yaml uses indentation (spaces) to determine the document hierarchy and the relationships of the sections to one another. The use of white space is significant. White space should not be used simply for formatting purposes, and tabs should not be used at all. - -Note: If the status parameters are not specified correctly, the utility does not issue a warning. For example, if the success parameter is misspelled and is set to true, a warning is not issued and an email is not sent to the email address after a successful operation. To ensure email notification is configured correctly, run tests with email notifications configured. - -This is the format of the gp_email_contacts.yaml YAML file for `gpbackup` email notifications: - -``` -contacts: - gpbackup: - - address: name@domain - status: - success: [true | false] - success_with_errors: [true | false] - failure: [true | false] - gprestore: - - address: name@domain - status: - success: [true | false] - success_with_errors: [true | false] - failure: [true | false] -``` - -##### Email YAML File Sections - -**contacts:** Required. The section that contains the `gpbackup` and `gprestore` sections. The YAML file can contain a `gpbackup` section, a `gprestore` section, or one of each. - -**gpbackup** : Optional. Begins the gpbackup email section. - -**address** : Required. At least one email address must be specified. Multiple email address parameters can be specified. Each address requires a status section. - -:::tip Note -name@domainis a single, valid email address. -::: - -**status** : Required. Specify when the utility sends an email to the specified email address. The default is to not send email notification. - -You specify sending email notifications based on the completion status of a backup or restore operation. At least one of these parameters must be specified and each parameter can appear at most once. - -**success** : Optional. Specify if an email is sent if the operation completes without errors. If the value is true, an email is sent if the operation completes without errors. If the value is false (the default), an email is not sent. - -**success_with_errors** : Optional. Specify if an email is sent if the operation completes with errors. If the value is true, an email is sent if the operation completes with errors. If the value is false (the default), an email is not sent. - -**failure** : Optional. Specify if an email is sent if the operation fails. If the value is true, an email is sent if the operation fails. If the value is false (the default), an email is not sent. - -**gprestore** : Optional. Begins the gprestore email section. This section contains the address and status parameters that are used to send an email notification after a `gprestore` operation. The syntax is the same as the gpbackup section. - -#### Examples - -This example YAML file specifies sending email to email addresses depending on the success or failure of an operation. For a backup operation, an email is sent to a different address depending on the success or failure of the backup operation. For a restore operation, an email is sent to single address only when the operation succeeds or completes with errors. - -``` -contacts: - gpbackup: - - address: dba_backup_success@whpg.io - status: - success:true - - address: dba_backup_failure@whpg.io - status: - success_with_errors: true - failure: true - gprestore: - - address: dba_restore_status@example.com - status: - success: true - success_with_errors: true -``` - - - -## Backup Files Details - -A complete backup set for `gpbackup` includes multiple metadata files, supporting files, and CSV data files, each designated with the timestamp at which the backup was created. - -By default, metadata and supporting files are stored on the WarehousePG coordinator host in the directory `$COORDINATOR_DATA_DIRECTORY/backups/YYYYMMDD/YYYYMMDDHHMMSS/`. If you specify a custom backup directory, this same file path is created as a subdirectory of the backup directory. The following table describes the names and contents of the metadata and supporting files. - -:::tip Warning: -All `gpbackup` metadata files are created with read-only permissions. Never delete or modify the metadata files for a `gpbackup` backup set. Doing so may render the backup files non-functional. -::: - -### Coordinator Metadata Files - -#### `gpbackup__metadata.sql` - -Contains global and database-specific metadata: - -- DDL for objects that are global to the WarehousePG cluster, and not owned by a specific database within the cluster. -- DDL for objects in the backed-up database (specified with --dbname) that must be created before to restoring the actual data, and DDL for objects that must be created after restoring the data. - -##### Global objects include: - -- Tablespaces -- Databases -- Database-wide configuration parameter settings (GUCs) -- Resource group definitions -- Resource queue definitions -- Roles -- GRANT assignments of roles to databases - -:::tip Note -Note: Global metadata is not restored by default. You must include the --with-globals option to the `gprestore` command to restore global metadata. -::: - -**Database-specific objects** -that must be created before to restoring the actual data include: - -- Session-level configuration parameter settings (GUCs) -- Schemas -- Procedural language extensions -- Types -- Sequences -- Functions -- Tables -- Protocols -- Operators and operator classes -- Conversions -- Aggregates -- Casts -- Views -- Materialized Views (only view definition is backed up / restored, not data) -- Constraints -- Database-specific objects that must be created after restoring the actual data include: -- Indexes -- Rules -- Triggers. (While WarehousePG does not support triggers, any trigger definitions that are present are backed up and restored.) - -example: - -``` -CREATE TABLE uat.refund ( - returned_date_sk integer, - returned_time_sk integer, - item_sk integer, - refunded_customer_sk integer, - refunded_cdemo_sk integer, - refunded_hdemo_sk integer, -) DISTRIBUTED RANDOMLY; - -ALTER TABLE uat.refund OWNER TO gpadmin; - -CREATE INDEX item_sk_cr_fee_idx ON uat.refund USING btree (item_sk, fee); - -CREATE INDEX item_sk_idx ON uat.refund USING btree (item_sk); -``` - -#### `gpbackup__toc.yaml` - -Table of contents file, containing location information of the various object DDL in the `gpbackup__metadata.sql` files - -This file also contains the table names and OIDs used for locating the corresponding table data in CSV data files that are created on each segment. - -example: - -``` -- schema: tpcds - name: household_demographics - oid: 37231 - attributestring: (hd_demo_sk,hd_income_band_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count) - rowscopied: 7200 - partitionroot: "" - isreplicated: false - distbyenum: false -- schema: tpcds - name: income_band - oid: 37238 - attributestring: (ib_income_band_sk,ib_lower_bound,ib_upper_bound) - rowscopied: 20 - partitionroot: "" - isreplicated: false - distbyenum: false -- schema: tpcds - name: inventory - oid: 37245 - attributestring: (inv_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand) - rowscopied: 11745000 - partitionroot: "" - isreplicated: false - distbyenum: false -``` - -See [Segment Data Files](#segfiles) - -#### `gpbackup__report` - -Contains information about the backup operation that is used to populate the email notice (if configured) that is sent after the backup completes. This file contains information such as: - -- Command-line options that were provided -- Database that was backed up -- Database version -- Backup type - -example: - -``` -WarehousePG Backup Report - -timestamp key: 20250530232002 -gpdb version: 6.27.1 build commit:ab5a612bfdc355ad2d601860dfb70a47778c8dd7 -gpbackup version: 1.30.5 - -database name: whpg_tpcds -command line: gpbackup --dbname whpg_tpcds --no-compression --single-data-file --backup-dir /tmp/whpg_tpcds_backups -compression: None -plugin executable: None -backup section: All Sections -object filtering: None -includes statistics: No -data file format: Single Data File Per Segment -incremental: False - -start time: Fri May 30 2025 23:20:02 -end time: Fri May 30 2025 23:20:13 -duration: 0:00:11 - -backup status: Success - -database size: 1699 MB -segment count: 3 - -count of database objects in backup: -aggregates 0 -casts 0 -collations 0 -constraints 0 -conversions 0 -default privileges 0 -database gucs 1 -event triggers 0 -extensions 0 -foreign data wrappers 0 -foreign servers 0 -functions 0 -indexes 2 -``` - -See [Setting Up Email Alerts](#email) - -#### `gpbackup__config.yaml` - -Contains metadata about the execution of the particular backup task, including: - -- `gpbackup` version -- Database name -- WarehousePG version -- Additional option settings such as `--no-compression`,` --compression-level`, `--metadata-only`, `--data-only`, and `--with-stats`. - -example: - -``` -backupdir: /tmp/whpg_tpcds_backups -backupversion: 1.30.5 -compressed: false -compressiontype: gzip -databasename: whpg_tpcds -databaseversion: 6.27.1 build commit:ab5a612bfdc355ad2d601860dfb70a47778c8dd7 -segmentcount: 8 -dataonly: false -datedeleted: "" -excluderelations: [] -excludeschemafiltered: false -excludeschemas: [] -excludetablefiltered: false -includerelations: [] -includeschemafiltered: false -includeschemas: [] -includetablefiltered: false -incremental: false -leafpartitiondata: false -metadataonly: false -plugin: "" -pluginversion: "" -restoreplan: -- timestamp: "20250530232002" - tablefqns: - - tpcds.call_center - - tpcds.catalog_page - - tpcds.catalog_returns - - tpcds.catalog_sales - - tpcds.customer - - tpcds.customer_address - - tpcds.customer_demographics -``` - - - -### Segment Data Files - -By default, each segment creates one compressed CSV file for each table that is backed up on the segment. You can optionally specify the `--single-data-file` option to create a single data file on each segment. The files are stored in `/backups/YYYYMMDD/YYYYMMDDHHMMSS/`. - -If you specify a custom backup directory, segment data files are copied to this same file path as a subdirectory of the backup directory. If you include the `--leaf-partition-data` option, `gpbackup` creates one data file for each leaf partition of a partitioned table, instead of just one table for file. - -By default, each table is backed up into a distinct file on the segment hosts: -The file uses a name format `gpbackup___.gz` where: - -``is the content ID of the segment. -`` is the timestamp of the `gpbackup` operation. -`` is the object ID of the table. - -#### default multiple data file example - -``` --rw------- 1 gpadmin gpadmin 216 May 30 23:46 gpbackup_1_20250530234643_32812.zst --rw------- 1 gpadmin gpadmin 177641 May 30 23:46 gpbackup_1_20250530234643_32822.zst --rw------- 1 gpadmin gpadmin 3080557 May 30 23:46 gpbackup_1_20250530234643_32829.zst --rw------- 1 gpadmin gpadmin 32449514 May 30 23:46 gpbackup_1_20250530234643_36476.zst --rw------- 1 gpadmin gpadmin 1989827 May 30 23:46 gpbackup_1_20250530234643_37203.zst --rw------- 1 gpadmin gpadmin 475551 May 30 23:46 gpbackup_1_20250530234643_37210.zst --rw------- 1 gpadmin gpadmin 3002507 May 30 23:46 gpbackup_1_20250530234643_37217.zst --rw------- 1 gpadmin gpadmin 535332 May 30 23:46 gpbackup_1_20250530234643_37224.zst --rw------- 1 gpadmin gpadmin 8895 May 30 23:46 gpbackup_1_20250530234643_37231.zst --rw------- 1 gpadmin gpadmin 74 May 30 23:46 gpbackup_1_20250530234643_37238.zst --rw------- 1 gpadmin gpadmin 19518846 May 30 23:46 gpbackup_1_20250530234643_37245.zst --rw------- 1 gpadmin gpadmin 606613 May 30 23:46 gpbackup_1_20250530234643_37459.zst --rw------- 1 gpadmin gpadmin 4595 May 30 23:46 gpbackup_1_20250530234643_37466.zst --rw------- 1 gpadmin gpadmin 177 May 30 23:46 gpbackup_1_20250530234643_37473.zst --rw------- 1 gpadmin gpadmin 152 May 30 23:46 gpbackup_1_20250530234643_37480.zst --rw------- 1 gpadmin gpadmin 327 May 30 23:46 gpbackup_1_20250530234643_37487.zst --rw------- 1 gpadmin gpadmin 4710343 May 30 23:46 gpbackup_1_20250530234643_37494.zst -``` - - **`--single-data-file`** example - -All table data for a particular segment is written to a single file: - -- `gpbackup__` - -With a corresponding table of contents files: - -- `gpbackup___toc.yaml` - -``` --rw-rw-r-- 1 gpadmin gpadmin 417025744 May 30 23:20 gpbackup_0_20250530232002 --r--r--r-- 1 gpadmin gpadmin 2035 May 30 23:20 gpbackup_0_20250530232002_toc.yaml -``` - -The metadata file `gpbackup__toc.yaml` references this `` to locate the data for a specific table in a schema. - -You can optionally specify the gzip compression level (from 1-9) using the `--compression-level` option, or disable compression entirely with `--no-compression`. If you do not specify a compression level, `gpbackup` uses compression level 1 by default. - - - -## Return Codes - -One of these codes is returned after `gpbackup` or `gprestore` completes. - -0 – Backup or restore completed with no problems -1 – Backup or restore completed with non-fatal errors. See log file for more information. -2 – Backup or restore failed with a fatal error. See log file for more information. - - - -## Supported WHPG versions - -The WarehousePG provided `gpbackup` and `gprestore` utilities are compatible with WarehousePG versions - -- 6.27.1 or later -- 7.2.1-WHPG or later - - - -## Limitations - -`gpbackup` and `gprestore` have the following limitations: - -If you create an index on a parent partitioned table, `gpbackup` does not back up that same index on child partitioned tables of the parent, as creating the same index on a child would cause an error. However, if you exchange a partition, `gpbackup` does not detect that the index on the exchanged partition is inherited from the new parent table. In this case, `gpbackup` backs up conflicting CREATE INDEX statements, which causes an error when you restore the backup set. - -You can execute multiple instances of `gpbackup`, but each execution requires a distinct timestamp. - -Database object filtering is currently limited to schemas and tables. - -When backing up a partitioned table where some or all leaf partitions are in different schemas from the root partition, the leaf partition table definitions, including the schemas, are backed up as metadata. This occurs even if the backup operation specifies that schemas that contain the leaf partitions should be excluded. To control data being backed up for this type of partitioned table in this situation, use the `--leaf-partition-data` option. - -If the `--leaf-partition-data` option is not specified, the leaf partition data is also backed up even if the backup operation specifies that the leaf partition schemas should excluded. -If the `--leaf-partition-data` option is specified, the leaf partition data is not be backed up if the backup operation specifies that the leaf partition schemas should excluded. Only the metadata for leaf partition tables are backed up. -If you use the `gpbackup` --single-data-file option to combine table backups into a single file per segment, you cannot perform a parallel restore operation with `gprestore` (cannot set --jobs to a value higher than 1). - -You cannot use the `--exclude-table-file` with `--leaf-partition-data`. Although you can specify leaf partition names in a file specified with `--exclude-table-file`, `gpbackup` ignores the partition names. - -Backing up a database with `gpbackup` while simultaneously running DDL commands might cause `gpbackup` to fail, in order to ensure consistency within the backup set. For example, if a table is dropped after the start of the backup operation, `gpbackup` exits and displays the error message ERROR: relation `` does not exist. - -`gpbackup` might fail when a table is dropped during a backup operation due to table locking issues. `gpbackup` generates a list of tables to back up and acquires an ACCESS SHARED lock on the tables. If an `EXCLUSIVE LOCK` is held on a table, `gpbackup` acquires the `ACCESS SHARED` lock after the existing lock is released. If the table no longer exists when `gpbackup` attempts to acquire a lock on the table, `gpbackup` exits with the error message. - -For tables that might be dropped during a backup, you can exclude the tables from a backup with a `gpbackup` table filtering option such as `--exclude-table` or --exclude-schema. - -A backup created with `gpbackup` can only be restored to a WarehousePG cluster with the same number of segment instances as the source cluster. If you run gpexpand to add segments to the cluster, backups you made before starting the expand cannot be restored after the expansion has completed. - -**Parent topic:** [WarehousePG Administrator Guide](../index.md) +The `gpbackup` and `gprestore` documentation has moved to the [WarehousePG Backup and Restore](../../../../whpg-backup/index.md) section. diff --git a/docs/whpg/7x/admin_guide/managing/backup-main.md b/docs/whpg/7x/admin_guide/managing/backup-main.md index 7ed9c77..8d47816 100644 --- a/docs/whpg/7x/admin_guide/managing/backup-main.md +++ b/docs/whpg/7x/admin_guide/managing/backup-main.md @@ -11,11 +11,7 @@ WarehousePG supports parallel and non-parallel methods for backing up and restor ## Parallel Backup with gpbackup and gprestore -`gpbackup` and `gprestore` are the recommended WarehousePG backup and restore utilities. `gpbackup` utilizes `ACCESS SHARE` locks at the individual table level, instead of `EXCLUSIVE` locks on the `pg_class` catalog table. This enables you to run DML statements during the backup, such as `CREATE`, `ALTER`, `DROP`, and `TRUNCATE` operations, as long as those operations do not target the current backup set. Backup files created with `gpbackup` are designed to provide future capabilities for restoring individual database objects along with their dependencies, such as functions and required user-defined datatypes. - -`gpbackup`, `gprestore`, and related utilities are provided as a separate download - -![Parallel Restore Using Parallel Backup Files](../../images/parallel_backup_restore.png "Parallel Restore Using Parallel Backup Files") +`gpbackup` and `gprestore` are the recommended WarehousePG backup and restore utilities, provided as a separate download. For documentation, see [WarehousePG Backup and Restore](../../../../whpg-backup/index.md). diff --git a/docs/whpg/7x/index.md b/docs/whpg/7x/index.md index 36c4f0e..4714f41 100644 --- a/docs/whpg/7x/index.md +++ b/docs/whpg/7x/index.md @@ -35,4 +35,4 @@ WarehousePG is based on **PostgreSQL open-source technology**. It is essentially [Security Guide](/docs/7x/security_guide/index.md) -[Backup & Restore Guide](/docs/7x/admin_guide/backup_restore/index.md) +[Backup & Restore Guide](/whpg-backup/index.md) diff --git a/docs/whpg/7x/ref_guide/utility_guide/reference/index.md b/docs/whpg/7x/ref_guide/utility_guide/reference/index.md index 6cd72df..efbe8b3 100644 --- a/docs/whpg/7x/ref_guide/utility_guide/reference/index.md +++ b/docs/whpg/7x/ref_guide/utility_guide/reference/index.md @@ -10,7 +10,6 @@ navigation: - gpactivatestandby - gpaddmirrors - gpbackup_manager - - gpbackup - gpcheckcat - gpcheckperf - gpconfig @@ -26,7 +25,6 @@ navigation: - gpmovemirrors - gprecoverseg - gpreload - - gprestore - gpssh - gpssh-exkeys - gpstart @@ -49,7 +47,9 @@ WarehousePG uses the standard PostgreSQL client and server programs and provides Several utilities are installed when you install the WarehousePG server. These utilities reside in `$GPHOME/bin`. -Other utilities such as clients and backup/restore may be found at [Warehouse PG Github Repo](https://github.com/warehouse-pg) +For the `gpbackup` and `gprestore` utilities, see [WarehousePG Backup and Restore](../../../../../whpg-backup/index.md). + +Other utilities such as clients may be found at [Warehouse PG Github Repo](https://github.com/warehouse-pg) WarehousePG provides the following utility programs. Superscripts identify those utilities that require separate downloads, as well as those utilities that are also installed with the Client and Loader Tools Packages. (See the Note following the table.) All utilities are installed when you install the WarehousePG server, unless specifically identified by a superscript. @@ -61,7 +61,6 @@ WarehousePG provides the following utility programs. Superscripts identify those - [dropuser](dropuser.md)1 - [gpactivatestandby](gpactivatestandby.md) - [gpaddmirrors](gpaddmirrors.md) -- [gpbackup](gpbackup.md)2 - [gpcheckcat](gpcheckcat.md) - [gpcheckperf](gpcheckperf.md) - [gpconfig](gpconfig.md) @@ -75,7 +74,6 @@ WarehousePG provides the following utility programs. Superscripts identify those - [gpmovemirrors](gpmovemirrors.md) - [gprecoverseg](gprecoverseg.md) - [gpreload](gpreload.md) -- [gprestore](gprestore.md)2 - [gpssh](gpssh.md) - [gpssh-exkeys](gpssh-exkeys.md) - [gpstart](gpstart.md) @@ -97,8 +95,6 @@ WarehousePG provides the following utility programs. Superscripts identify those > 1 The utility program is also installed with the **WarehousePG Client and Loader Tools Package** -> 2 The utility program can be obtained from the **WarehousePG Backup and Restore** - Additional utilities may be found at [Warehouse PG Github Org Repo](https://github.com/warehouse-pg) **Parent topic:** [WarehousePG Utility Guide](index.md)