whpg-backup documentation for OSS

docs/.vitepress/config.mts

@@ -33,7 +33,8 @@ export default defineConfig(
         text: 'Docs',
         items: [
           { text: '7.x', link: '/docs/7x' },
-          { text: '6.x', link: '/docs/6x' }
+          { text: '6.x', link: '/docs/6x' },
+          { text: 'Backup & restore', link: '/whpg-backup/' }
         ]
       },
       { text: 'GitHub', link: 'https://github.com/warehouse-pg/warehouse-pg' },
@@ -45,22 +46,22 @@ export default defineConfig(
 
     sidebar: [
       {
-        text: 'WHPG Documentation'
+        text: 'WHPG documentation'
       },
       {
         text: "WHPG 7.x",
         link: "/docs/7x/" ,
         collapsed: false,
         items: [
-          { text: "Release Notes", link: "/docs/7x/release_notes" },
-          { text: "Install Guide", link: "/docs/7x/install_guide/" },
-          { text: "Admin Guide", link: "/docs/7x/admin_guide/" },
-          { text: "Best Practices", link: "/docs/7x/best_practices/" },
-          { text: "Utility Guide", link: "/docs/7x/ref_guide/utility_guide/" },
-          { text: "Analytics Guide", link: "/docs/7x/admin_guide/analytics/" },
-          { text: "Reference Guide", link: "/docs/7x/ref_guide/" },
-          { text: "Security Guide", link: "/docs/7x/security_guide/" },
-          { text: "Backup & Restore Guide", link: "/docs/7x/admin_guide/backup_restore/" }
+          { text: "Release notes", link: "/docs/7x/release_notes" },
+          { text: "Install guide", link: "/docs/7x/install_guide/" },
+          { text: "Admin guide", link: "/docs/7x/admin_guide/" },
+          { text: "Best practices", link: "/docs/7x/best_practices/" },
+          { text: "Utility guide", link: "/docs/7x/ref_guide/utility_guide/" },
+          { text: "Analytics guide", link: "/docs/7x/admin_guide/analytics/" },
+          { text: "Reference guide", link: "/docs/7x/ref_guide/" },
+          { text: "Security guide", link: "/docs/7x/security_guide/" },
+          { text: "Backup & restore guide", link: "/whpg-backup/" }
 
 
 
@@ -73,17 +74,33 @@ export default defineConfig(
       link: "/docs/6x/index.html" ,
       collapsed: true,
       items: [
-          { text: "Release Notes", link: "/docs/6x/release_notes/" },
-          { text: "Install Guide", link: "/docs/6x/install_guide/" },
-          { text: "Admin Guide", link: "/docs/6x/admin_guide/" },
-          { text: "Best Practices", link: "/docs/6x/best_practices/" },
-          { text: "Utility Guide", link: "/docs/6x/ref_guide/utility_guide/" },
-          { text: "Analytics Guide", link: "/docs/6x/admin_guide/analytics/" },
-          { text: "Reference Guide", link: "/docs/6x/ref_guide/" },
-          { text: "Security Guide", link: "/docs/6x/security-guide/" },
+          { text: "Release notes", link: "/docs/6x/release_notes/" },
+          { text: "Install guide", link: "/docs/6x/install_guide/" },
+          { text: "Admin guide", link: "/docs/6x/admin_guide/" },
+          { text: "Best practices", link: "/docs/6x/best_practices/" },
+          { text: "Utility guide", link: "/docs/6x/ref_guide/utility_guide/" },
+          { text: "Analytics guide", link: "/docs/6x/admin_guide/analytics/" },
+          { text: "Reference guide", link: "/docs/6x/ref_guide/" },
+          { text: "Security guide", link: "/docs/6x/security-guide/" },
 
 
       ],
+      },
+      {
+        text: "WHPG backup & restore",
+        link: "/whpg-backup/" ,
+        collapsed: false,
+        items: [
+          {
+            text: "Release notes", link: "/whpg-backup/release_notes/"},
+          { text: "Overview", link: "/whpg-backup/overview/" },
+          { text: "Installing", link: "/whpg-backup/installing" },
+          { text: "Backing up and restoring", link: "/whpg-backup/using" },
+          { text: "Creating incremental backups", link: "/whpg-backup/incremental" },
+          { text: "Using the S3 storage plugin", link: "/whpg-backup/s3-plugin" },
+          { text: "Reference",
+            link: "/whpg-backup/reference/" }
+        ],
       }
     ]
 

docs/whpg-backup/incremental.md

@@ -0,0 +1,99 @@
+---
+title: Creating incremental backups of append-optimized tables
+
+---
+
+Use `gpbackup` and `gprestore` to create and restore incremental backups of append-optimized tables. Full backups contain every object in the database, making a restore, filtered or complete, straightforward, but they cost more in size and duration. Incremental backups reduce that cost when the total amount of changed append-optimized data is small compared to the data that hasn't changed since the last backup.
+
+`gpbackup` backs up an append-optimized table in an incremental backup only if one of these operations ran against the table after the last full or incremental backup:
+
+- `ALTER TABLE`
+- `DELETE`
+- `INSERT`
+- `TRUNCATE`
+- `UPDATE`
+- `DROP` and then re-create the table
+
+An incremental backup always includes every specified heap table, and backs up an append-optimized table, including a column-oriented table, only if it's changed. An incremental backup set consists of a full backup, plus the incremental backups that capture changes to the database since that full backup. For example, a full backup on Sunday plus three daily incremental backups on Monday, Wednesday, and Friday together form one backup set. Restoring from an incremental backup requires the complete backup set, and every backup in a set must use consistent command-line options so that `gpbackup` can build on it and `gprestore` can restore from it.
+
+::: info Note
+`gpbackup` always backs up a whole table when it includes it in an incremental backup, whether that's because it's a heap table or because an append-optimized table changed. It doesn't back up just the changed rows within a table. For a partitioned append-optimized table, though, `gpbackup` backs up only the leaf partitions that changed, not the entire table, so partitioning large append-optimized tables can substantially reduce the size of your incremental backups.
+:::
+
+## Creating an incremental backup set
+
+Start an incremental backup set with a full backup, then add incremental backups on top of it using a few required options, plus optional ones for more control. `gpbackup` checks that later backups in the set stay consistent with the ones already there.
+
+```bash
+gpbackup --dbname <database> --incremental --backup-dir <backup_directory> --leaf-partition-data
+```
+
+Include these options when you create an incremental backup:
+
+- **`--leaf-partition-data`**: Required for every backup in the set, including the full backup that serves as its base.
+- **`--incremental`**: Required to create an incremental backup. Not compatible with `--data-only` or `--metadata-only`.
+- **`--from-timestamp`** (optional): Use with `--incremental` to specify the timestamp of an existing backup, either an incremental backup or the initial full backup, to build on. The new backup must use the same command-line options as the backup specified. If you don't specify `--from-timestamp`, `gpbackup` looks for a compatible backup using the [backup history database](using.md#understanding-the-backup-history-database).
+
+When you add an incremental backup to a set, `gpbackup` checks that these options match across the full backup and all incremental backups in the set:
+
+- **`--dbname`**: The database must be the same.
+- **`--backup-dir`**: The full backup and its incremental backups must be in the same location.
+- **`--single-data-file`**: Must be either specified or absent for every backup in the set.
+- **`--plugin-config`**: If specified, it must be specified for every backup in the set, referencing the same plugin binary.
+- **`--include-table-file`, `--include-schema`**, and other [filtering options](using.md#filtering-backups-and-restores): Must be the same across the set. For schema filters, `gpbackup` checks only the schema names, not the objects they contain.
+- **`--no-compression`**: If specified, it must be specified for every backup in the set. If the full backup uses compression, the incremental backups must too, though they can use different compression levels. The default compression level is 1.
+
+::: warning
+If you try to add an incremental backup to a set and its command-line options aren't consistent with the rest of the set, the backup operation fails.
+:::
+
+::: warning
+Changing the WarehousePG segment configuration invalidates incremental backups. After you add or remove segment instances, create a full backup before creating another incremental backup.
+:::
+
+## Planning your backup strategy
+
+Decide how many full and incremental backups to retain based on your organization's regulatory requirements for backups. Factor in your Recovery Point Objective (RPO), when you design your backup strategy. Combine `--incremental` backups, effective partitioning of append-optimized/column-oriented (AO/CO) tables, and the `--include` and `--exclude` filters for a flexible and performant backup strategy.
+
+When you archive incremental backups, archive the complete backup set, including all files created on the coordinator and all segments.
+
+The `--with-stats` option doesn't need to be consistent across a backup set. However, to restore statistics with the `gprestore --with-stats` option, the backup you restore from must have been taken with `--with-stats`.
+
+### Example
+
+This example follows a common strategy, combining a weekly full backup with daily incremental backups. Each backup's file name includes a timestamp, in `YYYYMMDDhhmmss` format, identifying when `gpbackup` created it.
+
+1. Create the first full backup of the set on Sunday:
+
+  ```bash
+  gpbackup --dbname ww_sales --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data
+  ```
+
+1. Create an incremental backup based on that full backup each following day:
+
+  ```bash
+  gpbackup --dbname ww_sales --incremental --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data
+  ```
+
+Continuing this pattern from Monday through Saturday results in a set of seven backups: one full backup and six incremental backups, each identified by its own timestamp. At the end of a week-long set like this, continue in one of two ways:
+
+- Run another full backup the next Sunday to start a new backup set and retire or archive the previous week's set.
+- Aggregate the week into a single incremental backup using `--from-timestamp`, pointing at the previous Sunday's full backup:
+
+  ```bash
+  gpbackup --dbname ww_sales --incremental --from-timestamp 20250518010000 --backup-dir /nfsmount/whpg_backups/ww_sales --leaf-partition-data
+  ```
+
+  This new backup captures the same changes as the whole week's incremental backups combined, since it's incremental from the same full backup they were. That makes the week's individual incremental backups redundant, and you can delete them to reclaim storage space, leaving just the full backup and this one aggregated incremental backup to cover the entire week.
+
+## Restoring incremental backup sets
+
+To restore to a point in the middle of an incremental backup set, specify that backup's timestamp with `gprestore --timestamp`. For example, to restore to Wednesday's incremental backup from the weekly set described above:
+
+```bash
+gprestore --timestamp 20250521040000 --redirect-db ww_sales_pitr --create-db
+```
+
+Every earlier backup in the set, the Monday and Tuesday incremental backups and the Sunday full backup, must also be available for this restore to succeed.
+
+Restore operations can start from any backup in the set, but changes captured in incremental backups later than the one you restore from aren't restored. When restoring from an incremental backup set, `gprestore` restores each append-optimized table from its most recent version in the set. Heap tables are always restored from the latest backup in the set.

docs/whpg-backup/index.md

@@ -0,0 +1,14 @@
+---
+title: WarehousePG Backup and Restore
+
+---
+
+Use WarehousePG Backup and Restore (`whpg-backup`) to create and restore logical backups of your WarehousePG tables in parallel. By default, backups are written to local disk on the coordinator and segment hosts, though you can consolidate them in a shared directory or write them directly to S3-compatible storage. `whpg-backup` writes the metadata and DDL for a backup to the coordinator host, while each segment writes its own table data to CSV files, so backup and restore performance scales with your cluster.
+
+## Key capabilities
+
+- Parallel backup and restore across the coordinator and all segment hosts
+- Full database backups, plus [incremental backups](incremental.md) that include every heap table but back up append-optimized tables only if they changed
+- [Filtering](using.md#filtering-backups-and-restores) backups and restores by schema, table, or leaf partition
+- [Email notifications](using.md#setting-up-email-alerts) when a backup or restore completes
+- Backing up directly to [Amazon S3 or S3-compatible storage](s3-plugin.md)

docs/whpg-backup/installing.md

@@ -0,0 +1,72 @@
+---
+title: Installing WarehousePG Backup and Restore
+
+---
+
+`whpg-backup` provides the `gpbackup` and `gprestore` utilities. Build them from source, then install the binaries on every host in your WarehousePG cluster.
+
+## Prerequisites
+
+- **Build host:** [Go](https://go.dev/doc/install) 1.23 or later, plus `sqlite3`. Building directly on the WarehousePG coordinator avoids a cross-compile step, since it's already running the same OS as your segment hosts. If you build on a macOS workstation instead, use `make build_linux` to cross-compile for the cluster.
+- **Every host in the cluster:** `sqlite3`. `gpbackup` splits work between the coordinator and every segment host, so the `sqlite3` runtime dependency needs to be present wherever `gpbackup`, `gprestore`, or `gpbackup_helper` runs, not just on the build host.
+
+## Building from source
+
+1. Clone the repository and change into it:
+
+    ```bash
+    git clone https://github.com/warehouse-pg/whpg-backup.git
+    cd whpg-backup
+    ```
+
+1. Build the `gpbackup`, `gprestore`, and `gpbackup_helper` binaries:
+
+    ```bash
+    make depend
+    make build
+    ```
+
+    The binaries are placed in `$HOME/go/bin`. If you're cross-compiling from a macOS workstation, run `make build_linux` instead.
+
+:::: tip
+Add `$HOME/go/bin` to your `PATH` so you can run the binaries directly from the build host:
+
+```bash
+export PATH=$PATH:$HOME/go/bin
+```
+::::
+
+## Installing on the cluster
+
+1. Confirm `sqlite3` is installed on every host in the cluster, including the coordinator, standby coordinator, and all segment hosts. It's installed by default on many platforms, but install it if it's missing.
+
+1. On the coordinator, create a file `all_hosts` which lists all hosts in the WHPG cluster. For example:
+
+    ```ini
+    cdw
+    scdw
+    sdw1
+    sdw2
+    sdw3
+    ```
+
+1. Transfer the binaries to all hosts in the cluster and place them on `$PATH`, for example in `$GPHOME/bin`. Use the `gpsync` utility on WarehousePG 7, or the `gpscp` utility on WarehousePG 6:
+
+    ### WHPG 7
+
+    ```bash
+    gpsync -f all_hosts $HOME/go/bin/gpbackup $HOME/go/bin/gprestore $HOME/go/bin/gpbackup_helper =:$GPHOME/bin
+    ```
+
+    ### WHPG 6
+
+    ```bash
+    gpscp -f all_hosts $HOME/go/bin/gpbackup $HOME/go/bin/gprestore $HOME/go/bin/gpbackup_helper =:$GPHOME/bin
+    ```
+
+1. Verify the installation by checking the `gpbackup` and `gprestore` versions:
+
+    ```bash
+    gpbackup --version
+    gprestore --version
+    ```

docs/whpg-backup/overview/architecture.md

@@ -0,0 +1,46 @@
+---
+title: Architecture of WarehousePG Backup and Restore
+
+---
+
+WarehousePG Backup and Restore (`whpg-backup`) provides the `gpbackup` and `gprestore` utilities, which create and restore logical backups of WarehousePG tables in parallel. Because a WarehousePG cluster spreads data across many segment hosts, `gpbackup` and `gprestore` split backup and restore work across the cluster, so throughput scales with the number of segments. By default, backup files stay on local disk on the coordinator and segment hosts, though you can point them at a shared directory instead or write them directly to [S3-compatible storage](../s3-plugin.md).
+
+![Parallel restore using parallel backup files](../images/parallel_backup_restore.png "Parallel restore using parallel backup files")
+
+## How it works
+
+`gpbackup` creates a full backup of a single database, splitting the work between the coordinator and every segment host. The coordinator writes the backup's metadata and DDL, while each segment writes its own table data to CSV files, using the `COPY ... ON SEGMENT` command. Files are compressed by default; use `--no-compression` to disable it. Each `gpbackup` task runs in a single transaction and acquires an `ACCESS SHARE` lock on each table it backs up. On completion, `gpbackup` returns the backup's timestamp, which identifies that backup for a later restore.
+
+The metadata files contain everything `gprestore` needs to reconstruct a full backup set in parallel, restoring the coordinator's schema and database objects alongside each segment's table data. Because table data is stored as CSV, other utilities, such as [`gpload`](../../whpg/7x/ref_guide/utility_guide/reference/gpload.md), can also load it, in the same cluster or a different one.
+
+`gprestore` restores the database objects from a backup by default. Restoring the WarehousePG cluster's global objects, such as roles and tablespaces, is optional. See [Objects included in a backup](#objects-included-in-a-backup) for the distinction.
+
+For the full walkthrough with example commands and output, see [Running a full backup](../using.md#running-a-full-backup) and [Restoring a full backup](../using.md#restoring-a-full-backup).
+
+## Objects included in a backup
+
+`gpbackup` and `gprestore` back up and restore two categories of objects:
+
+- **Database objects** for the database you specify with the `--dbname` option. They include:
+  - **Schema and storage objects:** schemas, tables, views, materialized views (DDL only), sequences, indexes, types, and domains.
+  - **Programmability objects:** functions, procedural language extensions, operators, operator families, and operator classes, casts, conversions, aggregates, and text search parsers, dictionaries, templates, and configurations.
+  - **Rules and triggers:** rules, and triggers (definitions only; WarehousePG doesn't support triggers).
+  - **External tables:** readable and writable external tables (DDL only), and protocols.
+  - **Metadata and configuration:** comments, owners, extensions, session-level configuration parameter settings, and table statistics (only with the `--with-stats` option).
+
+  ::: info Note
+  `gpbackup` and `gprestore` never include these schemas in a backup set: `gp_toolkit`, `information_schema`, `pg_aoseg`, `pg_bitmapindex`, `pg_catalog`, `pg_toast*`, and `pg_temp*`.
+  :::
+
+- **Global objects** are also backed up by default. They include:
+  - Databases
+  - Database-wide configuration parameter settings
+  - `GRANT` assignments of roles to databases
+  - Resource group definitions
+  - Resource queue definitions
+  - Roles
+  - Tablespaces
+
+  ::: info Note
+  `gprestore` only restores global objects if you include the `--with-globals` option. Conversely, `gpbackup --without-globals` skips the backup of global objects.
+  :::

docs/whpg-backup/overview/compatibility.md

@@ -0,0 +1,11 @@
+---
+title: Compatibility for WarehousePG Backup and Restore
+
+---
+
+The WarehousePG-provided `gpbackup` and `gprestore` utilities are compatible with the following WarehousePG versions:
+
+- WarehousePG 6.27.1 or later
+- WarehousePG 7.2.1-WHPG or later
+
+`gpbackup` and `gprestore` support RHEL 7, 8, and 9, on `x86_64`.

docs/whpg-backup/overview/index.md

@@ -0,0 +1,10 @@
+---
+title: Overview of WarehousePG Backup and Restore
+
+---
+
+WarehousePG Backup and Restore (`whpg-backup`) creates and restores WarehousePG backup sets in parallel across the coordinator and all segment hosts.
+
+- [Architecture](architecture.md): Learn how `whpg-backup` coordinates backups and restores across the coordinator and segment hosts.
+- [Compatibility](compatibility.md): Check compatible WarehousePG versions and supported platforms.
+- [Known issues](known_issues.md): Review current limitations before you deploy.