diff --git a/.env.example b/.env.example index 252f5c68..3f9f402b 100644 --- a/.env.example +++ b/.env.example @@ -129,5 +129,8 @@ LOG_MAX_FILE="3" # TINYBIRD_TRACKER_TOKEN="p.eyJxxxxx" # TINYBIRD_ADMIN_TOKEN="p.eyJxxxxx" # TINYBIRD_WORKSPACE_ID="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" +# Shared secret Ghost uses to authenticate Tinybird sync requests to Traffic +# Analytics, for automation analytics. Generate one with `openssl rand -hex 32`. +# TINYBIRD_SYNC_AUTH="xxxxxxxx" # SALT_STORE_TYPE="file" # TRAFFIC_ANALYTICS_LOG_LEVEL="info" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 807fae57..18b7be76 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,9 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # The released main layout the migration tests read (GD_RELEASED_MAIN). + - run: git fetch --no-tags --depth=1 origin main:refs/remotes/origin/main + - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 with: node-version: "26" @@ -118,6 +121,19 @@ jobs: - name: Self-update end to end run: tests/e2e/self-update.sh + # An installation of the released main layout, migrated by the served + # launcher: refusals before any change, recovery after a failed start, and + # the migration keeping its project, data, routes and overrides. + migrate-main-linux: + name: Migration from the released main layout on Linux + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Migration end to end + run: tests/e2e/migrate-main.sh + # Real backups and restores on Docker Engine: a local site with ActivityPub # backed up, restored over itself and into a new directory, the lock # refusing a second operation, and a dump that fails. @@ -209,6 +225,7 @@ jobs: - launcher-linux - install-linux - self-update-linux + - migrate-main-linux - backup-linux - import-linux - launcher-macos diff --git a/.gitignore b/.gitignore index f455f042..a2c5e15c 100644 --- a/.gitignore +++ b/.gitignore @@ -98,7 +98,7 @@ caddy/custom/* caddy/global/* !caddy/global/.gitignore !caddy/global/README.md -# Operator's own Caddyfile from pre-1.0 installations (see the S6 migration) +# The released main layout's own Caddyfile, kept by its migration (docs/install.md) caddy/Caddyfile.local # The operator's own Compose overrides (docs/configuration.md) compose.override.yml diff --git a/README.md b/README.md index 194f2855..34fe068b 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,9 @@ Configuration to run Ghost and its services with Docker Compose. > **This is the `next-docker` development branch.** It is being rebuilt around > a manager image: a small CLI in a container, started by a bash launcher that > uses Docker. It installs local and production sites, imports local and production -> Ghost-CLI sites, backs up and restores them, and updates image installations -> between beta releases. Migration from the released `main` layout remains on -> the [roadmap](docs/ghost-cli-replacement.md). For a supported setup +> Ghost-CLI sites, backs up and restores them, updates image installations +> between beta releases, and moves installations of the released `main` layout +> onto it. Remaining work is on the [roadmap](docs/ghost-cli-replacement.md). For a supported setup > today, use the `main` branch. ## The launcher @@ -91,6 +91,7 @@ From `scripts/`, run `pnpm install --frozen-lockfile`, `pnpm run typecheck` and tests/e2e/launcher.sh # stand-in Docker, then the real image tests/e2e/install.sh # real installations: pulls images, binds 80 and 443 tests/e2e/self-update.sh # release updates, failed-update write retention and recovery +tests/e2e/migrate-main.sh # an installation of the released main layout, migrated tests/e2e/backup.sh # real backups and restores, including ActivityPub tests/e2e/import.sh # real Ghost-CLI sites exported and imported GD_TEST_HOST_CHANGES=1 tests/e2e/production-import.sh # a Ghost-CLI production site moved; changes the host diff --git a/TINYBIRD.md b/TINYBIRD.md index 6f7f3ac1..fbd33618 100644 --- a/TINYBIRD.md +++ b/TINYBIRD.md @@ -8,6 +8,7 @@ Note: Currently Traffic Analytics features are behind a feature flag. For now, y 1. Run `docker compose run --rm tinybird-deploy` and wait for the service to exit successfully. This will create your Tinybird datasources, pipes and API endpoints. It may take a minute or two to complete the first time. You should see "Deployment #1 is live!" in your terminal before the service exits. 1. Run `docker compose run --rm tinybird-login get-tokens` 1. Copy and paste the values from the previous step into your `.env` file (Tinybird credentials are operator settings, not Ghost application settings) +1. If using automations analytics, generate a shared sync secret with `openssl rand -hex 32` and add it to `.env`: `./ghost-docker config set .env TINYBIRD_SYNC_AUTH `. Ghost and Traffic Analytics share it; after adding or changing it, run `docker compose up -d` to recreate their containers (a restart alone does not apply environment changes). 1. Run `docker compose --profile=analytics up -d` to start all services in the background 1. Add `analytics` to `COMPOSE_PROFILES` in your `.env` file, alongside the site mode, to include the `analytics` profile automatically when running `docker compose` commands. Profiles are additive: adding `analytics` does not change the site mode. 1. In production, add the analytics route to `caddy/sites/site.caddy`, inside the site's block: `import /etc/caddy/snippets/TrafficAnalytics traffic-analytics-:3000`, then reload Caddy with `docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile` diff --git a/compose.yml b/compose.yml index 24467c53..5bb90b0b 100644 --- a/compose.yml +++ b/compose.yml @@ -65,6 +65,7 @@ services: tinybird__tracker__datasource: analytics_events tinybird__adminToken: ${TINYBIRD_ADMIN_TOKEN:-} tinybird__workspaceId: ${TINYBIRD_WORKSPACE_ID:-} + tinybird__sync_auth_key: ${TINYBIRD_SYNC_AUTH:-} tinybird__stats__endpoint: ${TINYBIRD_API_URL:-https://api.tinybird.co} volumes: - ${UPLOAD_LOCATION:-./data/ghost}:${GHOST_CONTENT_PATH:-/home/ghost/content} @@ -194,6 +195,7 @@ services: SALT_STORE_TYPE: ${SALT_STORE_TYPE:-file} SALT_STORE_FILE_PATH: /data/salts.json TINYBIRD_TRACKER_TOKEN: ${TINYBIRD_TRACKER_TOKEN:-} + TINYBIRD_SYNC_AUTH: ${TINYBIRD_SYNC_AUTH:-} LOG_LEVEL: ${TRAFFIC_ANALYTICS_LOG_LEVEL:-info} networks: ghost_network: diff --git a/docs/architecture.md b/docs/architecture.md index c35a4b56..58601651 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -257,4 +257,22 @@ useful fresh/damaged-install diagnostics. Bundle v1 has its own S12 must record the exact schema/format/launcher versions supported by the stable release here. Subsequent changes must be compatible or explicitly versioned with a migration or old-format reader; pinned launchers must still start newer update -managers. Migration from the released `main` layout remains required in S6b. +managers. + +### Migration from the released main layout + +`src/legacy.ts` is migration `0001-compose-profiles`: the one way an installation +of the released `main` layout (a clone with no metadata) reaches this layout, +entered through the served launcher's `self-update`. `src/legacy/caddy.ts` +carries the operator's Caddyfile as written, with main's environment variables +filled in and main's snippets kept beside it rather than translated into this +layout's shape; Caddy decides whether it loads. `src/legacy/config.ts` splits +its `.env`. +It decides everything before changing anything: Compose resolves the staged +configuration with the operator's overrides and Caddy loads the staged routes. +It then follows self-update's recovery boundary, reusing its snapshot, writer +pause and backup; the backup skips drift refusal because the stopped containers +deliberately ran main's images. The metadata is written last and is the only +record of completion: a site with it is never migrated again. There is no +general migration framework; a later migration from a released layout adds +what it needs. diff --git a/docs/caddy.md b/docs/caddy.md index 4b8c53d4..a838e602 100644 --- a/docs/caddy.md +++ b/docs/caddy.md @@ -25,6 +25,16 @@ import /etc/caddy/sites/*.caddy import /etc/caddy/custom/*.caddy ``` +## Routes moved from the released main layout + +A site moved from `main` keeps its own Caddyfile as `caddy/sites/site.caddy` +([install.md](install.md#moving-from-the-released-main-layout)). It imports +main's snippets, which take no arguments, from `caddy/sites/legacy-snippets/`, +and proxies to bare service names (`ghost:2368`), which resolve on the site's +own network. Both keep working. Moving to this layout's snippets and aliases, +as `install` writes them, is optional, and is needed before the site joins a +shared Caddy. + ## Changing routes Edit the file, then reload Caddy explicitly: diff --git a/docs/configuration.md b/docs/configuration.md index 7ec75e79..24cbf3e4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -295,8 +295,9 @@ minimum and exact source-version requirement. ## Existing installations Do not use `git pull` to move a released `main` installation to this layout. -That migration is still [roadmap work](ghost-cli-replacement.md#s6b--migration-from-the-released-main-layout). -Moving a Ghost-CLI site is a separate, supported import described in +The served launcher's `self-update` migrates it, splitting its `.env` into +`.env` and `ghost.env` as this page describes; [install.md](install.md#moving-from-the-released-main-layout) +lists what each setting becomes. Moving a Ghost-CLI site is a separate, supported import described in [install.md](install.md#importing-a-ghost-cli-site). ## Installed image pins diff --git a/docs/ghost-cli-replacement.md b/docs/ghost-cli-replacement.md index b5f2ed0a..4d44228f 100644 --- a/docs/ghost-cli-replacement.md +++ b/docs/ghost-cli-replacement.md @@ -11,8 +11,8 @@ Linear](https://linear.app/ghost/issue/PLA-412). | Outcome | Remaining work | Dependencies | | --- | --- | --- | -| Tagged single-site production | S6b legacy-layout migration, S5e production import, S12 qualification | Existing image self-update, local import, backup and restore | -| Several sites behind shared Caddy | S13 | Existing backup/restore; S6b for converting existing sites | +| Tagged single-site production | S5e production import, S12 qualification | Existing image self-update, local import, backup and restore | +| Several sites behind shared Caddy | S13 | Existing backup/restore and released-main migration | | Admin-driven Ghost upgrades | S7 host upgrade, S8 supervisor, S9 core adapter/API, S10 Admin | S7 before supervisor execution; S8 protocol before S9; S9 before S10; S13 integration if shipped | | Later extensions | S11 file secrets, S14 service references, S15 nightlies, S16 Redis | See each step | @@ -43,37 +43,6 @@ its recovery, and the admin domain in CI; the cross-host move is run by hand. Th `scripts/migrate.sh`; S12 must not merge `next-docker` into `main` before these scenarios pass. -### S6b — Migration from the released main layout - -Repo: ghost-docker. Depends on image self-update and backup/restore. This migration -remains required regardless of the development-format [compatibility -policy](architecture.md#compatibility), and gates merging into `main`. - -Implement ordered release migration scripts, recording completion in metadata when the -first migration needs that field. `0001-compose-profiles` must: - -- Handle both absent profiles and existing `analytics,activitypub`, adding - `production`; preserve credentials and project identity. -- Split Ghost application configuration into `ghost.env`, keeping operator - settings in `.env`; add `SITE_MODE`, `URL`, `PROJECT_DIR` and an exact pin for - the currently running Ghost version. -- Preserve the existing untracked Caddyfile before managed files are written. - Custom routes must still work with snippets that take explicit upstream and - domain arguments. Preserve supported customizations automatically, or stop - before changing the live site and explain what must be resolved. - -Existing installations have no launcher or metadata. The entry point is the served -launcher from inside the installation (`curl -fsSL https://docker.ghost.org/install.sh | -bash -s -- self-update`), not a raw `git pull` across the breaking change. This is a -migration of the released layout, distinct from self-update of a current `source: -checkout` site. - -Acceptance: absent metadata, an untagged starting commit, existing optional profiles, -custom Caddy routes and Compose overrides, skipped releases, repeated invocation and -failed hooks. Failures during migration, pulling or startup must follow the [recovery -boundary](architecture.md#recovery) and report the observed state accurately, retaining -any writes after startup. - ### S7 — Host-driven Ghost upgrades Repo: ghost-docker. Depends on backup/restore and release compatibility. Implement @@ -244,13 +213,15 @@ environment-based installs, older imports, restart, and restore still work. ### S12 — Single-site release qualification and documentation -Repo: ghost-docker, with cross-repo fixtures. Deps: S5e and S6b, plus existing install, -backup/restore and self-update; qualify S13, S7-S10 and S11 when they have shipped. +Repo: ghost-docker, with cross-repo fixtures. Deps: S5e, plus existing install, +backup/restore, self-update and released-main migration; qualify S13, S7-S10 and S11 when they have shipped. Gates the first stable tag and merging `next-docker` into `main`. Include the launcher on Linux, macOS and WSL2. Consolidate CI and qualify the actual minimum supported tools and image versions. Run fresh local/production install, optional-service variants, CLI migration, legacy stack update, Ghost upgrade/recovery, supervisor/Admin, and restore -scenarios. Include Linux runtime tests and macOS-compatible shell/configuration checks. +scenarios. Before merging, pin the migration tests' `GD_RELEASED_MAIN` +(default `origin/main`, in `manager/test/site.ts`, `tests/e2e/migrate-main.sh` and +the manager CI job) to `main`'s last commit, and rerun them. Include Linux runtime tests and macOS-compatible shell/configuration checks. Record the [compatibility baseline](architecture.md#compatibility): the exact metadata schema, backup format, launcher contract and bundle versions the stable release supports, and the obligations that hold for them from then on. @@ -265,7 +236,8 @@ shipped. ### S13 — Optional shared Caddy Repo: ghost-docker. Depends on backup/restore for new member sites; converting an -existing site into a member (PLA-504) also needs S6b. Several sites on one server, as +existing site into a member (PLA-504) starts from a site on this layout, which the +released-main migration provides. Several sites on one server, as Ghost-CLI ran several sites behind one nginx. 80 and 443 can belong to one Caddy only, so that Caddy is shared; everything else stays per site, MySQL included, so backup, restore, upgrade and import are unchanged. diff --git a/docs/install.md b/docs/install.md index bed6b7cb..138f7bfe 100644 --- a/docs/install.md +++ b/docs/install.md @@ -590,7 +590,8 @@ site's launcher starts that image, not the one it is pinned to. | `--channel CHANNEL` | The newest release on `stable` or `beta`, which the site then follows. | | `--to vX.Y.Z` | Exactly that release. | -The update refuses downgrades, incompatible Ghost versions, absent metadata, +The update refuses downgrades, incompatible Ghost versions, absent metadata +(except for [the released main layout](#moving-from-the-released-main-layout)), checkouts, another operation's lock and an unfinished update snapshot. It pulls images before the outage where possible, keeps a file snapshot in `.ghost-docker-update/`, then pauses Ghost and ActivityPub and takes a consistent @@ -655,6 +656,63 @@ before, and restore the backup (`./ghost-docker restore --yes backups/`) the backup records the commit checked out when it was taken, and restore refuses a clone at any other. +### Moving from the released main layout + +An installation made from `main` before this layout (a git clone with one +`.env`, a hand-edited `caddy/Caddyfile`, and no `ghost-docker` launcher) moves +onto it once, with the launcher as docker.ghost.org serves it, run in the +site directory: + +```bash +cd /path/to/your/ghost-docker +curl -fsSL https://docker.ghost.org/install.sh | bash -s -- self-update --check +curl -fsSL https://docker.ghost.org/install.sh | bash -s -- self-update +``` + +**Do not `git pull` across this change.** The new layout's Compose file selects +no services without the new settings, and a pull cannot write them. + +`self-update` finds no `.ghost-docker.json` and main's `.env`, and runs migration +`0001-compose-profiles`. Ghost and its database must be running, so it can +read the Ghost version the site runs and back the site up. Before changing +anything it works out and checks all of this, and stops, naming what to +resolve, if any of it cannot be carried over: + +| What | Becomes | +| --- | --- | +| `COMPOSE_PROFILES` | `production`, plus `analytics` and `activitypub` if they were on | +| Compose project | The same name, set as `COMPOSE_PROJECT_NAME`, so Caddy's certificates and every volume stay the site's | +| `DOMAIN`, `ADMIN_DOMAIN` | `URL`, `ADMIN_URL` | +| Credentials, ports, data locations, Tinybird settings | Kept in `.env`, with `SITE_MODE`, `PROJECT_DIR` and the rest of this layout's settings added | +| Ghost's configuration in `.env` (`mail__*`, `labs__*`, anything Ghost read) | Moved to `ghost.env`, with the values Ghost actually received. Keys the container sets itself (`url`, `database__*`) are listed and left out, as they were overridden on main too | +| `ghost:6-alpine` | The `next` image of **exactly the running version**, pinned by digest (`GHOST_IMAGE_REF`). Ghost 6.61.0 is the first with one: upgrade an older Ghost on main first (`docker compose pull ghost && docker compose up -d`) | +| `caddy/Caddyfile` | Kept as `caddy/Caddyfile.local`, and carried into `caddy/sites/site.caddy` as written: `{$DOMAIN}`, `{$ADMIN_DOMAIN}` and `{$ACTIVITYPUB_TARGET}` filled in, and `import snippets/...` pointed at main's snippets, kept in `caddy/sites/legacy-snippets/`. A leading global options block moves to `caddy/global/legacy.caddy`. Caddy loads the result before anything changes; if it does not load, nothing is changed | +| `compose.override.yml`, `GD_COMPOSE_OVERRIDES` | Kept, and resolved with the new layout and every `.env` setting before anything changes. A mount into `/var/lib/ghost`, where main's Ghost image kept its files, is refused: this layout's image keeps them in `/home/ghost/content`, and a backup holds content only in `data/ghost` | +| Stack files (`compose.yml`, snippets, `mysql-init/`) | The release's. Git must show them unedited: a change in the work tree or in a commit no remote has stops the migration. Move such changes into `compose.override.yml` or a `.caddy` file of your own afterwards | + +Then it keeps a copy of every file it writes in `.ghost-docker-update/`, stops +Ghost and ActivityPub, writes the new layout and `.ghost-docker.json`, takes a +checked backup into `backups/`, starts the site on the new layout and verifies +it through Caddy. Ghost is unavailable from the backup until the new layout +starts. + +Recovery follows [self-update](#self-update)'s: a failure before startup puts +main's files back and starts Ghost again in the same containers. A failure +after startup stops the services, puts main's files back, leaves the data as +it is, and says **the site needs you**. Ghost's version never changes, so +`docker compose up -d` starts main's layout on that data. The backup holds the +databases and content as they were before the migration started anything; +once the migration has run, `./ghost-docker restore --yes backups/` +puts them back. + +Afterwards the directory is a site installed from the image: update it with +`./ghost-docker self-update`, not git (its `.git` is left in place, and shows +the stack files as changed). Running the served command again is an ordinary +self-update. Compose files you used with `-f` by hand, such as +`compose.ipv6.yml`, are not seen by the manager: name them with +`GD_COMPOSE_OVERRIDES` when you migrate, as for every manager command +([configuration](configuration.md#the-compose-invocation-contract)). + ## backup and restore ```text diff --git a/manager/src/backup.ts b/manager/src/backup.ts index 5a7a0694..b29d389c 100644 --- a/manager/src/backup.ts +++ b/manager/src/backup.ts @@ -277,6 +277,14 @@ export interface BackupInput { * resumes them as soon as the capture is done. */ readonly pause?: WriterPause; + /** + * The configuration was just rewritten from the released main layout, with + * the writers stopped (legacy.ts): the stopped containers ran the old + * layout's images, which the configuration deliberately no longer names. + * Drift is not refused, and only services running exactly what the + * configuration names are recorded as having run it. + */ + readonly migrating?: boolean; } /** The manager taking the backup, as it was built. */ @@ -321,6 +329,7 @@ export async function takeBackup({ now = new Date(), consistent = false, pause, + migrating = false, }: BackupInput): Promise { const dir = site.dir; const databases = siteDatabases(site); @@ -329,7 +338,9 @@ export async function takeBackup({ const resolved = await io.busy('Resolving the Compose project', () => resolveSite(io, dir)); refuseMovedData(site, resolved); const overrides = siteOverrides(resolved); - await refuseDrift(io, resolved); + if (!migrating) { + await refuseDrift(io, resolved); + } const images: Record = {}; for (const [service, definition] of Object.entries(resolved.services)) { if (definition.image !== null) { @@ -381,7 +392,12 @@ export async function takeBackup({ // What runs, exactly: the backup records it beside what is configured, // and checks the dumps with the MySQL that wrote them. const running = await observeSite(io, resolved); - const ran = await runningImages(io, running); + const ran = await runningImages( + io, + migrating + ? running.filter((each) => resolved.services[each.service]?.image === each.image) + : running, + ); const commit = metadata.source === 'checkout' ? await checkedOut(io, dir) : null; io.stdout('\nThe capture\n'); diff --git a/manager/src/commands/install.ts b/manager/src/commands/install.ts index 2e19f0cb..b0befc22 100644 --- a/manager/src/commands/install.ts +++ b/manager/src/commands/install.ts @@ -496,7 +496,7 @@ async function sitePorts( } /** The first port no container publishes and, where that can be told, nothing on the host holds. */ -async function freeOnHost( +export async function freeOnHost( io: Io, published: ReadonlySet, start = DEFAULT_PORT, diff --git a/manager/src/commands/self-update.ts b/manager/src/commands/self-update.ts index cda812a8..5e2a3020 100644 --- a/manager/src/commands/self-update.ts +++ b/manager/src/commands/self-update.ts @@ -9,10 +9,18 @@ import { z } from 'zod'; import { defineCommand, flag } from '../command.ts'; import { findingErrors, validate } from '../config.ts'; import { CliError, describeError, EXIT } from '../errors.ts'; -import { atomicWrite, copyPresent } from '../fs.ts'; +import { atomicWrite } from '../fs.ts'; import type { Io } from '../io.ts'; import { acquireLock } from '../lock.ts'; -import { isoSeconds, requireMetadata, siteFiles, writeMetadata, type Metadata } from '../meta.ts'; +import { isReleasedMainLayout, migrateReleasedMain } from '../legacy.ts'; +import { + isoSeconds, + readMetadata, + requireMetadata, + siteFiles, + writeMetadata, + type Metadata, +} from '../meta.ts'; import { LAUNCHER, launcherContent, @@ -21,7 +29,7 @@ import { sha256, stackDir, } from '../payload.ts'; -import { describeServices, runningServices, stopServices } from '../recovery.ts'; +import { describeServices, runningServices, Snapshot, stopServices } from '../recovery.ts'; import { compareReleases, isRelease } from '../release.ts'; import { resolveSite } from '../resolved.ts'; import { heading, ok, printChecks } from '../report.ts'; @@ -210,43 +218,6 @@ function writePayloadChanges(io: Io, dir: string, stack: string, payload: Payloa } } -// --- The snapshot --------------------------------------------------------------- - -/** Copies of what an update may change, to put back if it fails. */ -class Snapshot { - readonly dir: string; - readonly root: string; - readonly paths: string[]; - - constructor(dir: string, paths: string[]) { - this.dir = dir; - this.root = join(dir, UPDATE_DIR); - this.paths = paths; - } - - take(): void { - mkdirSync(join(this.root, 'files'), { recursive: true, mode: 0o700 }); - copyPresent(this.dir, this.paths, join(this.root, 'files')); - } - - /** Every path as it was: copied back, or removed when it did not exist. */ - restore(): void { - for (const path of this.paths) { - const target = join(this.dir, path); - const kept = join(this.root, 'files', path); - rmSync(target, { recursive: true, force: true }); - if (existsSync(kept)) { - mkdirSync(dirname(target), { recursive: true, mode: 0o755 }); - cpSync(kept, target, { recursive: true, preserveTimestamps: true }); - } - } - } - - remove(): void { - rmSync(this.root, { recursive: true, force: true }); - } -} - // --- Deciding ------------------------------------------------------------------- type Direction = 'newer' | 'current' | 'downgrade'; @@ -300,6 +271,11 @@ export const selfUpdateCommand = defineCommand({ const requested = requestedRelease(flags.channel, flags.to, '--to'); const { context, site } = installedSite(io); const dir = site.dir; + // An installation of the released main layout has no metadata: this is + // how it moves onto this one (legacy.ts). + if (readMetadata(dir).state === 'absent' && isReleasedMainLayout(site.settings)) { + return migrateReleasedMain(io, context, site, requested, flags.check); + } const metadata = requireMetadata(dir, 'update cannot tell which files are its own'); if (metadata.source === 'checkout') { throw new CliError( diff --git a/manager/src/compose.ts b/manager/src/compose.ts index 7c65c812..83dc8b34 100644 --- a/manager/src/compose.ts +++ b/manager/src/compose.ts @@ -365,8 +365,12 @@ const variables = z.record(z.string(), z.unknown()); * and every override in use, merged, so one an override removes the only use * of is not among them. `$$` is a literal and is not among them either. */ -export async function composeVariables(io: Io, dir: string): Promise | null> { - const result = await compose(io, { dir })`config --variables --format json`; +export async function composeVariables( + io: Io, + dir: string, + inputs?: ComposeInputs, +): Promise | null> { + const result = await compose(io, { dir, inputs })`config --variables --format json`; if (result.exitCode !== 0) { return null; } diff --git a/manager/src/legacy.ts b/manager/src/legacy.ts new file mode 100644 index 00000000..bbfeef4d --- /dev/null +++ b/manager/src/legacy.ts @@ -0,0 +1,932 @@ +// Migration 0001-compose-profiles: an installation of the released `main` +// layout, moved onto this one. It is how such a site comes to `self-update` +// (docs/install.md#moving-from-the-released-main-layout). +// +// A `main` installation is a git clone that updates with `git pull`. It has no +// launcher and no metadata; its compose.yml has no site-mode profiles; its one +// `.env` is both Compose's settings and Ghost's configuration; its Caddyfile +// is an untracked, hand-edited file; it runs a Ghost-CLI-layout Ghost image. +// The served launcher, run in it as `self-update`, starts this release's +// manager, which finds no metadata and that layout, and does the following: +// +// 1. Works out everything, changing nothing: the project's name, as Compose +// and the running containers have it, so volumes and certificates stay +// the site's; the Ghost version that runs, and the `next` image of +// exactly it; the new `.env`, `ghost.env` and routes, resolved by Compose +// with the operator's overrides and loaded by Caddy. Anything it cannot +// carry over stops it here, saying what to resolve. +// 2. Keeps a copy of every file it will write, stops Ghost (and +// ActivityPub), writes the release's files and the site's own, and takes +// a checked backup of the databases and content. +// 3. Starts the site on the new layout and verifies it. +// +// Recovery follows self-update's (docs/architecture.md#recovery): before +// startup is attempted the old files are put back and Ghost started again as +// it was; after, the services are stopped, the old files put back, and the +// data left as it is, for the operator. +// +// Completion is recorded by what it writes: the metadata. A site with it is +// never migrated again, so a second run is an ordinary self-update. +import { + copyFileSync, + cpSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, +} from 'node:fs'; +import { dirname, join, relative } from 'node:path'; +import { refuseMovedData, siteOverrides, takeBackup } from './backup.ts'; +import { renderRoutes, SITE_FILE } from './caddy.ts'; +import { freeOnHost } from './commands/install.ts'; +import { releaseOf, type ManagerRelease, type Requested } from './commands/common.ts'; +import { + compose, + composeConfig, + composeError, + composeVariables, + upAndWait, + type ComposeInputs, +} from './compose.ts'; +import { documentedVariables, findingErrors, validate } from './config.ts'; +import type { Context } from './context.ts'; +import { inspectImage, runOnce } from './docker/client.ts'; +import * as env from './env.ts'; +import { CliError, describeError, EXIT } from './errors.ts'; +import { atomicWrite, PRIVATE, readIfExists } from './fs.ts'; +import { MINIMUM_IMPORT_VERSION, resolveExactGhost, type ResolvedGhost } from './ghost.ts'; +import type { Io } from './io.ts'; +import { + GLOBAL_FILE, + KEPT_CADDYFILE, + LEGACY_CADDYFILE, + carryCaddyfile, + fillEnvironment, + LEGACY_SNIPPETS, +} from './legacy/caddy.ts'; +import { + readLegacyEnv, + refuseInterpolated, + splitLegacyEnv, + type SplitEnv, +} from './legacy/config.ts'; +import { acquireLock } from './lock.ts'; +import { isoSeconds, SCHEMA_VERSION, writeMetadata, type Metadata } from './meta.ts'; +import { + LAUNCHER, + launcherContent, + managerPin, + payloadFiles, + sha256, + stackDir, +} from './payload.ts'; +import { takenPorts } from './ports.ts'; +import { git } from './process.ts'; +import { describeServices, runningServices, Snapshot, stopServices } from './recovery.ts'; +import { observeSite, resolveConfig, resolveSite, type ResolvedSite } from './resolved.ts'; +import { heading, ok, printChecks } from './report.ts'; +import { + COMPOSE_FILE, + ENV_EXAMPLE_FILE, + ENV_FILE, + GHOST_ENV_FILE, + hostOf, + META_FILE, + OPERATOR_FILES, + readSettings, + siteFacts, + splitProfiles, + UPDATE_DIR, + type SiteFacts, + type SiteSettings, +} from './site.ts'; +import { verifySite } from './verify.ts'; +import { atLeast } from './versions.ts'; +import { WriterPause } from './writers.ts'; + +export const MIGRATION = '0001-compose-profiles'; +const FROM = 'the released main layout'; + +/** How long pulling the release's images may take. */ +const PULL_MS = 30 * 60 * 1000; +/** The optional profiles `main` had. */ +const MAIN_PROFILES = ['analytics', 'activitypub']; +const HOSTED_ACTIVITYPUB = 'https://ap.ghost.org'; +/** Where main's Ghost-CLI-layout image is installed, its content included. */ +const MAIN_GHOST_INSTALL = '/var/lib/ghost'; +/** Where preflight stages the new `.env` and Caddy files, inside the site so the daemon sees them. */ +const STAGED = 'staged'; + +/** + * A site with no metadata whose `.env` is `main`'s: a DOMAIN, and none of the + * settings every site of this layout has. + */ +export const isReleasedMainLayout = (settings: SiteSettings): boolean => + settings.get('DOMAIN') !== undefined && + settings.get('URL') === undefined && + settings.get('SITE_MODE') === undefined; + +function refuse(message: string): never { + throw new CliError(`${message}\n Nothing has been changed.`); +} + +/** Everything decided before the first change. */ +interface Plan { + readonly io: Io; + readonly dir: string; + readonly release: ManagerRelease; + /** The manager image the site's launcher is pinned to. */ + readonly pin: string; + readonly project: string; + readonly profiles: string[]; + readonly domain: string; + readonly adminDomain: string; + readonly ghost: ResolvedGhost; + /** The services running before the migration, which it stops and may start again. */ + readonly running: string[]; + readonly stack: string; + /** The release's files, relative to the stack. */ + readonly files: string[]; + readonly env: string; + readonly ghostEnv: string; + readonly split: SplitEnv; + readonly routes: string; + readonly global: string | null; + /** main's snippets, by name, for LEGACY_SNIPPETS. */ + readonly snippets: Readonly>; + readonly hadCaddyfile: boolean; + /** Overrides other than compose.override.yml, relative to the site. */ + readonly overrides: string[]; + readonly port: number; +} + +/** + * `self-update` in a site of the released main layout. Returns the exit + * status, or throws a CliError. + */ +export async function migrateReleasedMain( + io: Io, + context: Context, + site: SiteFacts, + requested: Requested, + check: boolean, +): Promise { + const dir = site.dir; + io.stdout( + `${dir} is an installation of ${FROM}, with no ${META_FILE}.\n` + + `Moving it onto this release's layout (migration ${MIGRATION}).\n`, + ); + if (context.source !== 'image') { + refuse( + 'this manager was built from a checkout, and the migration writes the release’s files\n' + + ' from a published manager image. Run the served launcher in the site directory:\n' + + ' curl -fsSL https://docker.ghost.org/install.sh | bash -s -- self-update', + ); + } + // Held from before anything is staged, --check included: the staging + // directory is the one the snapshot is kept in, and a second run must + // never remove another's. + const lock = acquireLock(dir, `migration ${MIGRATION} from ${FROM}`); + try { + if (existsSync(join(dir, UPDATE_DIR))) { + refuse( + `${join(dir, UPDATE_DIR)} is left from a migration or update that did not finish, and holds\n` + + ' the files it would have put back. Once the site is as it should be, remove it and run this again.', + ); + } + const staged = join(dir, UPDATE_DIR, STAGED); + let plan: Plan; + try { + plan = await prepare(io, context, site, requested, staged); + } finally { + rmSync(join(dir, UPDATE_DIR), { recursive: true, force: true }); + } + return check ? report(plan) : await apply(plan); + } finally { + lock.release(); + } +} + +// --- Working it out -------------------------------------------------------------- + +async function prepare( + io: Io, + context: Context, + site: SiteFacts, + requested: Requested, + staged: string, +): Promise { + const dir = site.dir; + const release = releaseOf(requested, io.env); + const text = readFileSync(join(dir, ENV_FILE), 'utf8'); + const legacy = readLegacyEnv(text); + const old = legacy.values; + if (old.COMPOSE_FILE !== undefined) { + refuse( + '.env sets COMPOSE_FILE, which the manager does not use: it names Compose files itself.\n' + + ' Remove it, and name any extra file with GD_COMPOSE_OVERRIDES when you run this again\n' + + ' (docs/configuration.md#the-compose-invocation-contract).', + ); + } + const domain = (old.DOMAIN ?? '').toLowerCase(); + const adminDomain = (old.ADMIN_DOMAIN ?? '').toLowerCase(); + for (const [key, value] of [ + ['DOMAIN', domain], + ['ADMIN_DOMAIN', adminDomain], + ] as const) { + if ((key === 'DOMAIN' || value !== '') && hostOf(`https://${value}/`) !== value) { + refuse(`.env's ${key} is ${JSON.stringify(value)}, which is not a domain.`); + } + } + const oldProfiles = splitProfiles(old.COMPOSE_PROFILES ?? ''); + const unknown = oldProfiles.filter((profile) => !MAIN_PROFILES.includes(profile)); + if (unknown.length > 0) { + refuse( + `COMPOSE_PROFILES in .env names ${unknown.join(', ')}; ${FROM} had only ${MAIN_PROFILES.join(' and ')}.`, + ); + } + for (const key of ['DATABASE_PASSWORD', 'DATABASE_ROOT_PASSWORD']) { + if (!old[key]) { + refuse(`.env has no ${key}, which ${FROM} required.`); + } + } + + heading(io, `Checking the installation`); + const stack = stackDir(io.env); + const files = payloadFiles(stack); + await refuseEditedStack(io, dir, files); + ok(io, 'stack files', 'unedited, so the release’s replace them'); + + // The old layout, as Compose resolves it with the operator's overrides: + // the project's name, and what Ghost received. + const before = await io.busy('Resolving the Compose project', () => resolveSite(io, dir)); + const project = before.project; + const oldConfig = await composeConfig(io, dir); + const oldVariables = await composeVariables(io, dir); + if (project === '' || !oldConfig.ok || oldVariables === null) { + refuse( + 'Compose cannot resolve the installation as it is: check it with docker compose config.', + ); + } + const received = Object.fromEntries( + Object.entries(oldConfig.project.services.ghost?.environment ?? {}).map(([key, value]) => [ + key, + value ?? '', + ]), + ); + const overrides = siteOverrides(before); + ok(io, 'project', `${project}, kept, with its volumes and Caddy’s certificates`); + + const containers = await observeSite(io, before); + const running = containers + .filter((each) => each.state === 'running') + .map((each) => each.service); + const ghostContainer = containers.find( + (each) => each.service === 'ghost' && each.state === 'running', + ); + if (ghostContainer === undefined || !running.includes('db')) { + refuse( + 'Ghost and its database must be running, so the version Ghost runs can be kept and its data\n' + + ' backed up. Start the site as it is (docker compose up -d), then run this again.', + ); + } + const image = await inspectImage(io.docker, ghostContainer.imageId); + const version = image?.env.GHOST_VERSION; + if (!image?.repoDigests.some((digest) => digest.startsWith('ghost@')) || !version) { + refuse( + `Ghost runs ${ghostContainer.image}, which is not the official ghost image, so there is no image of\n` + + ' this layout to move it to. Run the official image (remove the override that changes it).', + ); + } + if (!atLeast(version, MINIMUM_IMPORT_VERSION)) { + refuse( + `this site runs Ghost ${version}, and this layout's images start at Ghost ${MINIMUM_IMPORT_VERSION}.\n` + + ' A migration never changes Ghost. Upgrade it on the layout it runs now, then run this again:\n' + + ' docker compose pull ghost && docker compose up -d', + ); + } + ok(io, 'ghost', `${version}, running`); + + heading(io, 'Resolving the images'); + const ghost = await io.busy(`Resolving the image of Ghost ${version}`, () => + resolveExactGhost(io, version), + ); + ok( + io, + 'ghost', + `${ghost.image}:${ghost.tag}, ${ghost.reference}: the same version, this layout's image`, + ); + const pin = await io.busy('Resolving the manager image', () => managerPin(io, context)); + + // Ghost was not published on main; here it is, on the loopback interface. + const port = await freeOnHost( + io, + (await takenPorts(io)).published, + Number(old.GHOST_PORT) > 0 ? Number(old.GHOST_PORT) : undefined, + ); + const profiles = ['production', ...oldProfiles]; + const generated: [string, string][] = [ + ['COMPOSE_PROFILES', profiles.join(',')], + ['SITE_MODE', 'production'], + ['COMPOSE_PROJECT_NAME', project], + ['PROJECT_DIR', dir], + ['NODE_ENV', 'production'], + ['URL', `https://${domain}`], + ...(adminDomain ? [['ADMIN_URL', `https://${adminDomain}`] as [string, string]] : []), + ['GHOST_IMAGE', ghost.image], + ['GHOST_VERSION', ghost.tag], + ['GHOST_IMAGE_REF', ghost.reference], + ['GHOST_CONTENT_PATH', ghost.contentPath], + ['GHOST_TINYBIRD_PATH', ghost.tinybirdPath], + ['GHOST_PORT', String(port)], + ['RESTART_POLICY', 'unless-stopped'], + ['HTTP_PORT', old.HTTP_PORT || '80'], + ['HTTPS_PORT', old.HTTPS_PORT || '443'], + ['DATABASE_HOST', 'db'], + ['DATABASE_PORT', '3306'], + ['DATABASE_NAME', 'ghost'], + ['DATABASE_USER', old.DATABASE_USER || 'ghost'], + ['DATABASE_PASSWORD', old.DATABASE_PASSWORD!], + ['DATABASE_ROOT_PASSWORD', old.DATABASE_ROOT_PASSWORD!], + ['DATABASE_EXTRA_DATABASES', 'activitypub'], + ['UPLOAD_LOCATION', old.UPLOAD_LOCATION || './data/ghost'], + ['MYSQL_DATA_LOCATION', old.MYSQL_DATA_LOCATION || './data/mysql'], + ]; + + // The new layout, as Compose would resolve it with these settings and the + // operator's overrides, staged where Compose and the daemon can read it. + mkdirSync(staged, { recursive: true, mode: 0o700 }); + const inputs: ComposeInputs = { + files: [join(stack, COMPOSE_FILE), ...before.files.slice(1)], + envFile: join(staged, ENV_FILE), + }; + // With every setting .env had, so an override requiring one of them + // (`${SMTP_HOST:?...}`) resolves as it does now. + const generatedKeys = new Set(generated.map(([key]) => key)); + atomicWrite( + inputs.envFile, + env.serializeAll([ + ...generated, + ...legacy.keys + .filter((key) => !generatedKeys.has(key)) + .map((key): [string, string] => [key, old[key]!]), + ]), + PRIVATE, + ); + const draft = await composeConfig(io, dir, inputs); + const newVariables = await composeVariables(io, dir, inputs); + if (!draft.ok || newVariables === null) { + refuse( + `Compose cannot resolve this release's layout with the site's settings and overrides:\n ${draft.ok ? 'its variables could not be listed' : draft.reason}`, + ); + } + const documented = new Set([ + ...documentedVariables(readIfExists(join(dir, ENV_EXAMPLE_FILE)) ?? ''), + ...documentedVariables(readFileSync(join(stack, ENV_EXAMPLE_FILE), 'utf8')), + ]); + const isOperatorKey = (key: string) => + key.startsWith('COMPOSE_') || + oldVariables.has(key) || + newVariables.has(key) || + documented.has(key); + refuseInterpolated(text, new Set(legacy.keys.filter(isOperatorKey))); + const split = splitLegacyEnv({ + legacy, + generated: generatedKeys, + isOperatorKey, + isInterpolated: (key) => newVariables.has(key), + container: new Set(Object.keys(draft.project.services.ghost?.environment ?? {})), + received, + }); + const envText = env.serializeAll( + [...generated, ...split.operator], + `# Site settings, moved from ${FROM} by migration ${MIGRATION}. See .env.example\n` + + '# for the optional ones. Write values with ./ghost-docker config set .env KEY VALUE,\n' + + '# which encodes them for Compose. Ghost’s own configuration is in ghost.env.\n', + ); + atomicWrite(inputs.envFile, envText, PRIVATE); + const ghostEnv = env.serializeAll( + split.ghost, + `# Ghost application settings for ${project}, moved from .env by migration\n` + + `# ${MIGRATION}; see ghost.env.example. Write values with ./ghost-docker config set\n` + + '# ghost.env KEY VALUE, which encodes them for Compose.\n\n', + ); + + const after = await resolveConfig(io, dir, inputs); + if (after.services.ghost?.image !== ghost.reference) { + refuse( + `with the site's overrides, the ghost service would run ${after.services.ghost?.image ?? 'nothing'}, not\n` + + ` ${ghost.reference}. Remove the override that sets its image, then run this again.`, + ); + } + // Where main's Ghost image was installed. This layout's image never looks + // there, so whatever an override mounts there would silently be lost to + // Ghost, and to the backup. + const stranded = (after.services.ghost?.mounts ?? []).filter( + (mount) => + mount.target === MAIN_GHOST_INSTALL || + mount.target.startsWith(`${MAIN_GHOST_INSTALL}/`), + ); + if (stranded.length > 0) { + refuse( + `the site's overrides mount ${stranded.map((mount) => `${mount.source} at ${mount.target}`).join(', ')} in the\n` + + ` ghost service. That is where main's Ghost image kept its files; this layout's keeps them in\n` + + ` ${ghost.contentPath}, and a backup holds content only in ./data/ghost. Move what the mount holds\n` + + ' into data/ghost (or remove it), remove the mount from the override, then run this again.', + ); + } + const draftSite = siteFacts(dir, { + get: (key) => env.toRecord(envText)[key], + }); + try { + refuseMovedData(draftSite, after); + } catch (error) { + refuse( + `${(error as Error).message.replace(/ Nothing has been changed\.$/, '')}\n` + + ' The migration backs the site up first, and a backup handles data only in ./data.', + ); + } + ok(io, 'configuration', `.env and ghost.env for a production site, resolved by Compose`); + + // The routes, loaded by the Caddy image the site will run. ActivityPub + // goes where main sent it, which with the profile on was still the hosted + // service unless ACTIVITYPUB_TARGET said otherwise: moving it would move + // the site's followers. + const values = { + domain, + adminDomain, + activitypub: old.ACTIVITYPUB_TARGET || HOSTED_ACTIVITYPUB, + }; + const original = readIfExists(join(dir, LEGACY_CADDYFILE)); + let routes: string; + let global: string | null = null; + // main's snippets, read before the release replaces them; git has shown them unedited. + const snippets: Record = {}; + if (original === undefined) { + routes = renderRoutes({ + project, + domain, + adminDomain, + email: '', + activitypub: /^(?:https?:\/\/)?activitypub:8080$/.test(values.activitypub), + }); + } else { + ({ site: routes, global } = carryCaddyfile(original, values)); + const kept = join(dir, 'caddy', 'snippets'); + for (const name of existsSync(kept) ? readdirSync(kept) : []) { + snippets[name] = fillEnvironment(readFileSync(join(kept, name), 'utf8'), values); + } + } + await validateRoutes(io, staged, stack, after, routes, global, snippets); + ok( + io, + SITE_FILE, + original === undefined + ? 'written: there was no caddy/Caddyfile' + : 'carried over from caddy/Caddyfile, and loaded by Caddy', + ); + + return { + io, + dir, + release, + pin, + project, + profiles, + domain, + adminDomain, + ghost, + running: [...new Set(running)], + stack, + files, + env: envText, + ghostEnv, + split, + routes, + global, + snippets, + hadCaddyfile: original !== undefined, + overrides, + port, + }; +} + +/** + * The release replaces every stack file `main` tracked, so one the operator + * changed would be lost. Git, which installed the site, says which they are: + * changed in the work tree, or by a commit no remote has. + */ +async function refuseEditedStack(io: Io, dir: string, files: readonly string[]): Promise { + const top = await git(io, dir, ['rev-parse', '--show-toplevel']); + if (!top.ok || top.stdout.trim() !== dir) { + refuse( + `${dir} is not a git checkout. ${FROM} is installed with git clone, and without its history it\n` + + ' cannot be told whether compose.yml or the Caddy snippets were edited, which the release replaces.', + ); + } + const tracked = await git(io, dir, ['ls-files', '--', ...files]); + const paths = tracked.stdout.split('\n').filter(Boolean); + if (!tracked.ok || paths.length === 0) { + refuse( + 'git does not list the stack’s files in this checkout, so it is not one of ' + + FROM + + '.', + ); + } + const status = await git(io, dir, [ + 'status', + '--porcelain', + '--untracked-files=no', + '--', + ...paths, + ]); + const remotes = await git(io, dir, ['for-each-ref', '--format=%(refname)', 'refs/remotes']); + if (!status.ok || !remotes.ok) { + refuse(`git cannot read the checkout: ${status.stderr || remotes.stderr}`); + } + if (remotes.stdout.trim() === '') { + refuse( + 'the checkout has no remote-tracking branches, so its own commits cannot be told apart from\n' + + ' released ones. Fetch from https://github.com/TryGhost/ghost-docker.git, then run this again.', + ); + } + const local = await git(io, dir, [ + 'log', + '--format=', + '--name-only', + 'HEAD', + '--not', + '--remotes', + '--', + ...paths, + ]); + const edited = [ + ...new Set([ + ...status.stdout + .split('\n') + .filter((line) => line.trim() !== '') + .map((line) => line.slice(3)), + ...local.stdout.split('\n').filter(Boolean), + ]), + ].sort(); + if (edited.length > 0) { + refuse( + `these files of the stack were changed here, and the release replaces them:\n` + + edited.map((file) => ` ${file}\n`).join('') + + ' Move each change into compose.override.yml (Compose) or a .caddy file of your own\n' + + ' (docs/caddy.md) once the site is migrated, put the files back (git checkout -- ),\n' + + ' and run this again.', + ); + } +} + +/** + * The routes as the stack's Caddyfile imports them, loaded by `caddy + * validate` in the image the site will run, with no network. What it refuses, + * a reload would too. + */ +async function validateRoutes( + io: Io, + staged: string, + stack: string, + after: ResolvedSite, + routes: string, + global: string | null, + snippets: Readonly>, +): Promise { + const caddy = join(staged, 'caddy'); + cpSync(join(stack, 'caddy'), caddy, { recursive: true }); + atomicWrite(join(staged, SITE_FILE), routes, 0o644); + writeSnippets(staged, snippets); + if (global !== null) { + atomicWrite(join(staged, GLOBAL_FILE), global, 0o644); + } + const image = after.services.caddy?.image; + if (!image) { + refuse('Compose resolves no caddy service for the production layout.'); + } + const pulled = await io.busy( + 'Pulling the release’s images, while the site keeps running', + () => + compose(io, { + dir: after.dir, + inputs: { files: after.files, envFile: join(staged, ENV_FILE) }, + timeout: PULL_MS, + })`pull --quiet --ignore-buildable`, + ); + if (pulled.exitCode !== 0) { + refuse(`the release's images could not be pulled: ${composeError(pulled)}`); + } + const validated = await io.busy('Loading the routes in Caddy', () => + runOnce(io.docker, { + image, + entrypoint: ['caddy'], + cmd: ['validate', '--config', '/etc/caddy/Caddyfile', '--adapter', 'caddyfile'], + binds: [{ source: caddy, target: '/etc/caddy', readOnly: true }], + network: 'none', + }), + ); + if (validated.status !== 0) { + const said = (validated.stderr || validated.stdout).trim().split('\n').slice(-6); + refuse( + 'Caddy does not load the routes carried over from caddy/Caddyfile:\n' + + said.map((line) => ` ${line}\n`).join('') + + ' They are carried as written, with DOMAIN, ADMIN_DOMAIN and ACTIVITYPUB_TARGET filled in\n' + + " and main's snippets kept beside them. Change caddy/Caddyfile so they load, then run\n" + + ' this again.', + ); + } +} + +/** main's snippets beside the carried routes, which import them. */ +function writeSnippets(root: string, snippets: Readonly>): void { + for (const [name, content] of Object.entries(snippets)) { + mkdirSync(join(root, LEGACY_SNIPPETS), { recursive: true, mode: 0o755 }); + atomicWrite(join(root, LEGACY_SNIPPETS, name), content, 0o644); + } +} + +// --- --check ----------------------------------------------------------------------- + +function report(plan: Plan): number { + const { io, split } = plan; + io.stdout( + [ + '', + `The migration would move this site onto ${describeRelease(plan.release)}, and change nothing else:`, + ` Ghost ${plan.ghost.version}, from the image of this layout: ${plan.ghost.reference}`, + ` Project ${plan.project} (${plan.profiles.join(',')}), its volumes kept`, + ` .env operator settings; ${split.operator.length} carried besides the generated ones`, + ` ghost.env ${split.ghost.length} Ghost settings, from .env`, + ...split.dropped.map( + ({ key, reason }) => ` ${key} not carried: ${reason}`, + ), + ` Routes ${SITE_FILE}${plan.hadCaddyfile ? ', from caddy/Caddyfile, kept as caddy/Caddyfile.local' : ''}`, + ` Loopback Ghost published on 127.0.0.1:${plan.port}`, + ` Launcher ./${LAUNCHER}, pinned to ${plan.pin}`, + '', + 'Ghost would be stopped for a backup, then the site started on the new layout.', + '', + ].join('\n'), + ); + return EXIT.ok; +} + +const describeRelease = (release: ManagerRelease): string => + release.version ?? 'this manager’s release'; + +// --- Changing it ------------------------------------------------------------------- + +/** Every path the migration writes, relative to the site: what its snapshot holds. */ +function touched(plan: Plan): string[] { + return [ + ...new Set([...OPERATOR_FILES, ...plan.overrides, ...plan.files, LAUNCHER, KEPT_CADDYFILE]), + ]; +} + +async function apply(plan: Plan): Promise { + const { io, dir } = plan; + heading(io, 'Keeping the current files'); + const snapshot = new Snapshot(dir, touched(plan)); + snapshot.take(); + ok(io, UPDATE_DIR, `.env, the Caddyfile, and the files the migration writes`); + + const pause = new WriterPause(io, dir, 'the migration'); + let servicesChanged = false; + let backup: string | null = null; + try { + heading(io, 'Stopping Ghost'); + await pause.stop(plan.running); + + heading(io, 'Writing the new layout'); + const metadata = write(plan); + + const findings = await io.busy('Validating the configuration', () => validate(io, dir)); + const errors = findingErrors(findings); + if (errors) { + throw new CliError(`the migrated configuration does not validate:\n${errors}`); + } + ok(io, 'configuration', 'valid'); + + heading(io, 'Backing up the site'); + backup = await takeBackup({ + io, + site: siteFacts(dir, readSettings(dir)!), + metadata, + consistent: true, + pause, + migrating: true, + }); + ok(io, 'backup', `${relative(dir, backup)}, checked`); + + heading(io, 'Starting the site'); + const pull = await io.busy( + 'Pulling the images this release names', + () => compose(io, { dir, timeout: PULL_MS })`pull --quiet --ignore-buildable`, + ); + if (pull.exitCode !== 0) { + throw new CliError(`the images could not be pulled: ${composeError(pull)}`); + } + // Set before attempting up: even a failed start may migrate data or accept writes. + servicesChanged = true; + pause.end(); + await upAndWait(io, dir, 'Starting the services and waiting for them to be healthy'); + ok(io, 'services', 'healthy, by their own health checks'); + + await verifySite(io, dir); + } catch (error) { + return recover(plan, snapshot, pause, backup, servicesChanged, error); + } + snapshot.remove(); + summarize(plan, backup!); + return EXIT.ok; +} + +/** The release's files, the site's own, and the metadata, written last. */ +function write(plan: Plan): Metadata { + const { io, dir, stack, files } = plan; + const checksums: Record = {}; + + if (plan.hadCaddyfile) { + copyFileSync(join(dir, LEGACY_CADDYFILE), join(dir, KEPT_CADDYFILE)); + ok(io, KEPT_CADDYFILE, 'your caddy/Caddyfile, kept for reference'); + } + for (const file of files) { + const source = join(stack, file); + const target = join(dir, file); + const content = readFileSync(source); + mkdirSync(dirname(target), { recursive: true, mode: 0o755 }); + atomicWrite(target, content, statSync(source).mode & 0o777); + checksums[file] = sha256(content); + } + ok(io, 'stack files', `${files.length} files of ${describeRelease(plan.release)}`); + const launcher = launcherContent(io.env, { image: plan.pin, channel: plan.release.channel }); + atomicWrite(join(dir, LAUNCHER), launcher, 0o755); + checksums[LAUNCHER] = sha256(launcher); + ok(io, LAUNCHER, `the launcher, pinned to ${plan.pin}`); + + atomicWrite(join(dir, ENV_FILE), plan.env, PRIVATE); + ok(io, ENV_FILE, 'operator settings, with the same credentials and project'); + atomicWrite(join(dir, GHOST_ENV_FILE), plan.ghostEnv, PRIVATE); + ok(io, GHOST_ENV_FILE, `${plan.split.ghost.length} Ghost settings, moved from .env`); + for (const { key, reason } of plan.split.dropped) { + printChecks(io, [{ status: 'note', label: 'not carried', detail: `${key}: ${reason}` }]); + } + atomicWrite(join(dir, SITE_FILE), plan.routes, 0o644); + writeSnippets(dir, plan.snippets); + if (plan.global !== null) { + atomicWrite(join(dir, GLOBAL_FILE), plan.global, 0o644); + } + ok(io, SITE_FILE, 'the site’s routes; yours to edit'); + + const metadata: Metadata = { + schemaVersion: SCHEMA_VERSION, + installedAt: isoSeconds(), + updatedAt: null, + mode: 'production', + channel: plan.release.channel, + source: 'image', + stack: { + version: plan.release.version, + ref: plan.release.version, + image: plan.pin, + previous: null, + }, + site: { + project: plan.project, + dir, + url: `https://${plan.domain}`, + domain: plan.domain, + adminDomain: plan.adminDomain || null, + }, + ghost: { + image: plan.ghost.image, + tag: plan.ghost.tag, + version: plan.ghost.version, + digest: plan.ghost.digest, + }, + profiles: plan.profiles, + payload: checksums, + }; + writeMetadata(dir, metadata); + ok(io, META_FILE, `installation metadata: migration ${MIGRATION} is done once this is written`); + return metadata; +} + +/** Before startup: the old files back, Ghost started again. After: the operator decides. */ +async function recover( + plan: Plan, + snapshot: Snapshot, + pause: WriterPause, + backup: string | null, + servicesChanged: boolean, + error: unknown, +): Promise { + const { io, dir } = plan; + io.stderr(`\n${describeError(error)}\n`); + io.stderr(`\nThe migration did not complete. Putting ${FROM}'s files back\n`); + const problems: string[] = []; + if (servicesChanged) { + const stopped = await stopServices(io, dir, 'Stopping the migrated site'); + if (stopped.error !== null) { + problems.push(`the services could not be stopped: ${stopped.error}`); + } + } + if (problems.length === 0) { + try { + snapshot.restore(); + } catch (restoreError) { + problems.push(`the files could not be put back: ${(restoreError as Error).message}`); + } + } + const resumed = [...pause.paused]; + if (problems.length === 0 && !servicesChanged) { + try { + await pause.resume(); + } catch (resumeError) { + problems.push((resumeError as Error).message); + } + } + + const kept = + backup === null + ? [] + : [`The backup taken for the migration is kept in ${relative(dir, backup)}.`]; + if (servicesChanged || problems.length > 0) { + const running = await runningServices(io, dir); + const filesBack = problems.length === 0; + io.stderr( + [ + '', + 'The site needs you.', + ...problems.map((problem) => ` ${problem}`), + ...(servicesChanged + ? [ + 'The new layout’s services started before the migration failed, and Ghost may have accepted', + `writes since the backup. Nothing was loaded over them. Ghost ${plan.ghost.version} ran on the data`, + 'either way: the migration never changes Ghost’s version.', + ] + : []), + describeServices(running), + ...(filesBack ? [`The files are ${FROM}'s again.`] : []), + '', + ...(filesBack + ? [ + 'Start the site as it was, on its data as it is: docker compose up -d', + ...(backup === null + ? [] + : [ + `The databases and content as they were before the migration started anything are in`, + `${relative(dir, backup)} (checked). Once the cause is fixed and the migration has run,`, + `./ghost-docker restore --yes ${relative(dir, backup)} puts them back.`, + ]), + ] + : kept), + '', + `The files as they were before the migration are in ${snapshot.root}/files.`, + `Once the site is as it should be, remove ${snapshot.root}.`, + '', + ].join('\n'), + ); + throw new CliError('the migration failed, and the site needs the operator.'); + } + snapshot.remove(); + io.stderr( + [ + '', + `Restored: the site is on ${FROM} again, with its files as they were` + + (resumed.length > 0 + ? `. ${resumed.join(' and ')}, stopped for the migration, ${resumed.length > 1 ? 'are' : 'is'} running again.` + : '.'), + ...kept, + '', + ].join('\n'), + ); + throw new CliError(`the migration failed; ${FROM} was restored.`); +} + +function summarize(plan: Plan, backup: string): void { + const { io, dir } = plan; + io.stdout( + [ + '', + `Moved from ${FROM} to ${describeRelease(plan.release)}.`, + '', + ` Site https://${plan.domain}`, + ` Ghost ${plan.ghost.version}, ${plan.ghost.reference}, the same version`, + ` Project ${plan.project} (${plan.profiles.join(',')})`, + ` Loopback 127.0.0.1:${plan.port}`, + ` Backup ${relative(dir, backup)}, of the site before it started on the new layout`, + '', + 'From now on:', + ' - Update the stack with ./ghost-docker self-update, never git pull: this directory', + ' is now installed from the manager image, and its stack files are the release’s.', + ' - Ghost’s own settings are in ghost.env; .env holds the rest.', + ` - The routes are in ${SITE_FILE}${plan.hadCaddyfile ? `; your old Caddyfile is ${KEPT_CADDYFILE}` : ''}.`, + ...(plan.global === null ? [] : [` - Its global options are in ${GLOBAL_FILE}.`]), + '', + ].join('\n'), + ); +} diff --git a/manager/src/legacy/caddy.ts b/manager/src/legacy/caddy.ts new file mode 100644 index 00000000..7d19aaeb --- /dev/null +++ b/manager/src/legacy/caddy.ts @@ -0,0 +1,91 @@ +// The operator's own Caddyfile from the released `main` layout, carried into +// this layout's routes as it is (docs/install.md#moving-from-the-released-main-layout). +// +// On `main`, caddy/Caddyfile was copied from Caddyfile.example and edited by +// hand. It read DOMAIN, ADMIN_DOMAIN and ACTIVITYPUB_TARGET from Caddy's +// environment and imported main's snippets, which take no arguments, by +// relative path. Here Caddy has no environment and the snippets take +// arguments. Rather than rewrite the operator's routes into this layout's +// shape, the file is kept as written: the three variables are filled in, and +// its snippet imports point at main's snippets, kept beside it. Bare service +// upstreams (`ghost:2368`) still resolve on the site's own network. Caddy +// itself then decides whether the result loads, before anything is changed. +import { join } from 'node:path'; + +/** The original, kept beside the routes it became; .gitignore leaves it out. */ +export const LEGACY_CADDYFILE = join('caddy', 'Caddyfile'); +export const KEPT_CADDYFILE = join('caddy', 'Caddyfile.local'); +/** main's snippets, which the carried routes import. Not `*.caddy`, so never loaded as sites. */ +export const LEGACY_SNIPPETS = join('caddy', 'sites', 'legacy-snippets'); +/** A global options block, which this layout's Caddyfile imports from caddy/global/. */ +export const GLOBAL_FILE = join('caddy', 'global', 'legacy.caddy'); + +/** What the old Caddy container's environment held, as main's compose.yml set it. */ +export interface LegacyCaddyValues { + readonly domain: string; + readonly adminDomain: string; + readonly activitypub: string; +} + +export interface CarriedCaddyfile { + /** caddy/sites/site.caddy. */ + readonly site: string; + /** A leading global options block's body, for GLOBAL_FILE; null when there was none. */ + readonly global: string | null; +} + +/** + * `{$NAME}` and `{$NAME:default}`, filled in when the file is loaded, and + * `{env.NAME}`, when a request is served: both read Caddy's environment, + * which no longer holds these three. Anything else is left as written. + */ +export function fillEnvironment(text: string, values: LegacyCaddyValues): string { + const known: Record = { + DOMAIN: values.domain, + ADMIN_DOMAIN: values.adminDomain, + ACTIVITYPUB_TARGET: values.activitypub, + }; + return text + .replace(/\{\$([A-Za-z_][A-Za-z0-9_]*)(?::([^}]*))?\}/g, (whole, name: string, fallback) => + name in known ? known[name] || (fallback ?? '') : whole, + ) + .replace(/\{env\.([A-Za-z_][A-Za-z0-9_]*)\}/g, (whole, name: string) => + name in known ? known[name]! : whole, + ); +} + +/** + * The operator's Caddyfile as caddy/sites/site.caddy. A global options block + * must come first in Caddy's whole configuration, which the stack's + * Caddyfile opens with its own, so one written first, `{` and `}` alone on + * their lines, moves to caddy/global/. Whatever else does not load, Caddy says. + */ +export function carryCaddyfile(text: string, values: LegacyCaddyValues): CarriedCaddyfile { + let routes = fillEnvironment(text.replace(/\r\n/g, '\n'), values).replace( + /^([ \t]*import[ \t]+)(?:\.\/)?snippets\//gm, + `$1/etc/caddy/sites/legacy-snippets/`, + ); + let global: string | null = null; + const lines = routes.split('\n'); + const first = lines.findIndex((line) => line.trim() !== '' && !line.trim().startsWith('#')); + if (first >= 0 && lines[first]!.trim() === '{') { + const close = lines.findIndex((line, index) => index > first && line === '}'); + if (close > first) { + global = `# Global options carried over from caddy/Caddyfile.\n${lines + .slice(first + 1, close) + .map((line) => line.replace(/^\t/, '')) + .join('\n') + .trim()}\n`; + routes = [...lines.slice(0, first), ...lines.slice(close + 1)].join('\n'); + } + } + const header = [ + '# Routes carried over from caddy/Caddyfile, which is kept as caddy/Caddyfile.local.', + '# They import the released main layout’s snippets, kept in legacy-snippets/ beside', + '# this file. This file is yours: edit it, then reload Caddy:', + '#', + '# docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile', + '', + ]; + return { site: `${[...header, routes.trim()].join('\n')}\n`, global }; +} diff --git a/manager/src/legacy/config.ts b/manager/src/legacy/config.ts new file mode 100644 index 00000000..76e4fe35 --- /dev/null +++ b/manager/src/legacy/config.ts @@ -0,0 +1,111 @@ +// The released `main` layout's single `.env`, split into this layout's `.env` +// and `ghost.env` (docs/install.md#moving-from-the-released-main-layout). +// +// On `main`, `.env` was both Compose's interpolation source and the ghost +// service's env_file, so every key in it reached Ghost. Here operator settings +// stay in `.env`, which Ghost never sees, and Ghost's own configuration moves +// to `ghost.env`. Which is which is decided the way `config validate` decides +// it (config.ts): a key Compose interpolates, or an example documents, is an +// operator setting; anything else was there for Ghost. +import * as env from '../env.ts'; +import { CliError } from '../errors.ts'; +import { isContainerOwned } from '../import/config.ts'; + +/** + * Keys `main` read that this layout replaces: URL and ADMIN_URL, and the + * ActivityPub upstream now written into the Caddy route. Kept only when an + * override of the operator's still interpolates them. + */ +export const SUPERSEDED = ['DOMAIN', 'ADMIN_DOMAIN', 'ACTIVITYPUB_TARGET'] as const; + +/** The old `.env`, read as Compose read it. */ +export interface LegacyEnv { + readonly values: Readonly>; + /** Every key, in the order of its first assignment. */ + readonly keys: readonly string[]; +} + +export function readLegacyEnv(text: string): LegacyEnv { + const multiline = env.scan(text).filter((assignment) => assignment.quoting === 'multiline'); + if (multiline.length > 0) { + throw new CliError( + `.env has values spanning several lines (${multiline.map(({ key }) => key).join(', ')}), which this\n` + + ' migration does not carry over. Put each on one line, then run it again. Nothing has been changed.', + ); + } + return { values: env.toRecord(text), keys: env.keys(text) }; +} + +export interface SplitInput { + readonly legacy: LegacyEnv; + /** Keys this migration writes itself, from what it has worked out. */ + readonly generated: ReadonlySet; + /** Is the key an operator setting, in the old layout or this one? */ + readonly isOperatorKey: (key: string) => boolean; + /** Does this layout's compose.yml, with the operator's overrides, still interpolate it? */ + readonly isInterpolated: (key: string) => boolean; + /** What this layout's ghost service sets itself, by key. */ + readonly container: ReadonlySet; + /** + * What Ghost actually received for each key on `main`, as Compose resolved + * it: env_file values are interpolated, so this, not the text, is the value. + */ + readonly received: Readonly>; +} + +export interface SplitEnv { + /** Operator settings carried into `.env`, besides the generated ones. */ + readonly operator: [string, string][]; + /** Ghost configuration for `ghost.env`. */ + readonly ghost: [string, string][]; + /** Keys left out, with why. Names only: values may be credentials. */ + readonly dropped: { key: string; reason: string }[]; +} + +export function splitLegacyEnv({ + legacy, + generated, + isOperatorKey, + isInterpolated, + container, + received, +}: SplitInput): SplitEnv { + const operator: [string, string][] = []; + const ghost: [string, string][] = []; + const dropped: { key: string; reason: string }[] = []; + for (const key of legacy.keys) { + const value = legacy.values[key]!; + if (generated.has(key)) { + continue; + } + if ((SUPERSEDED as readonly string[]).includes(key) && !isInterpolated(key)) { + dropped.push({ key, reason: 'replaced by URL, ADMIN_URL and the Caddy routes' }); + } else if (key.startsWith('COMPOSE_') || isOperatorKey(key)) { + operator.push([key, value]); + } else if (isContainerOwned(key, container)) { + dropped.push({ + key, + reason: 'set by the container, which took precedence on main too', + }); + } else { + ghost.push([key, received[key] ?? value]); + } + } + return { operator, ghost, dropped }; +} + +/** + * Operator settings whose value Compose would interpolate: a `$` that is not + * doubled. What Compose made of it is not knowable from the text alone, so the + * operator says what was meant, rather than this migration guessing. + */ +export function refuseInterpolated(text: string, operatorKeys: ReadonlySet): void { + const keys = env.lint(text).filter((key) => operatorKeys.has(key)); + if (keys.length > 0) { + throw new CliError( + `.env has values with an unescaped $ (${keys.join(', ')}), which Compose interpolates, so what\n` + + ' they hold cannot be carried over exactly. Write each literal $ as $$, check the site still\n' + + ' starts (docker compose up -d), then run this again. Nothing has been changed.', + ); + } +} diff --git a/manager/src/recovery.ts b/manager/src/recovery.ts index 0128e355..85bf75c9 100644 --- a/manager/src/recovery.ts +++ b/manager/src/recovery.ts @@ -13,7 +13,7 @@ import { copyPresent, sameTree } from './fs.ts'; import { checkRows, loadFile } from './import/database.ts'; import type { Io } from './io.ts'; import { ok } from './report.ts'; -import { DATA_DIRS, readSettings } from './site.ts'; +import { DATA_DIRS, readSettings, UPDATE_DIR } from './site.ts'; // --- The services ----------------------------------------------------------------- @@ -60,6 +60,47 @@ export function describeServices(running: string[] | null): string { return `These of its services are still running: ${running.join(', ')}.`; } +// --- Putting files back --------------------------------------------------------- + +/** + * Copies of the files an operation may change, kept in UPDATE_DIR, to put + * back if it fails before it may no longer: self-update and the migration + * from the released main layout. + */ +export class Snapshot { + readonly dir: string; + readonly root: string; + readonly paths: string[]; + + constructor(dir: string, paths: string[]) { + this.dir = dir; + this.root = join(dir, UPDATE_DIR); + this.paths = paths; + } + + take(): void { + mkdirSync(join(this.root, 'files'), { recursive: true, mode: 0o700 }); + copyPresent(this.dir, this.paths, join(this.root, 'files')); + } + + /** Every path as it was: copied back, or removed when it did not exist. */ + restore(): void { + for (const path of this.paths) { + const target = join(this.dir, path); + const kept = join(this.root, 'files', path); + rmSync(target, { recursive: true, force: true }); + if (existsSync(kept)) { + mkdirSync(dirname(target), { recursive: true, mode: 0o755 }); + cpSync(kept, target, { recursive: true, preserveTimestamps: true }); + } + } + } + + remove(): void { + rmSync(this.root, { recursive: true, force: true }); + } +} + // --- Setting a site aside ------------------------------------------------------- /** diff --git a/manager/test/legacy.test.ts b/manager/test/legacy.test.ts new file mode 100644 index 00000000..106af52a --- /dev/null +++ b/manager/test/legacy.test.ts @@ -0,0 +1,184 @@ +// Migration 0001-compose-profiles: what it makes of the released main +// layout's Caddyfile and `.env`. What Caddy and Compose make of the result is +// tests/e2e/migrate-main.sh. +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, test } from 'node:test'; +import { carryCaddyfile, fillEnvironment, type LegacyCaddyValues } from '../src/legacy/caddy.ts'; +import { readLegacyEnv, refuseInterpolated, splitLegacyEnv } from '../src/legacy/config.ts'; +import { releasedMain } from './site.ts'; + +const MAIN = releasedMain(); +const EXAMPLE = readFileSync(join(MAIN, 'caddy', 'Caddyfile.example'), 'utf8'); + +const VALUES: LegacyCaddyValues = { + domain: 'example.com', + adminDomain: '', + activitypub: 'https://ap.ghost.org', +}; +const carry = (text: string, values: Partial = {}) => + carryCaddyfile(text, { ...VALUES, ...values }); + +/** The lines that are not comments: what Caddy loads. */ +const code = (text: string) => + text + .split('\n') + .filter((line) => line.trim() !== '' && !line.trim().startsWith('#')) + .join('\n'); + +describe('the Caddyfile', () => { + test('the example is carried as written, with its variables filled in and its snippets kept', () => { + const { site, global } = carry(EXAMPLE); + const loaded = code(site); + assert.match(loaded, /^example\.com \{$/m); + for (const name of ['Logging', 'TrafficAnalytics', 'ActivityPub', 'SecurityHeaders']) { + assert.match( + loaded, + new RegExp(`^\\timport /etc/caddy/sites/legacy-snippets/${name}$`, 'm'), + ); + } + // Bare upstreams resolve on the site's own network. + assert.match(loaded, /^\t\treverse_proxy ghost:2368$/m); + assert.doesNotMatch(loaded, /\{\$|import snippets\//); + assert.equal(global, null); + // Everything else, comments included, as it was. + assert.equal( + code(site), + code( + EXAMPLE.replaceAll('{$DOMAIN}', 'example.com').replace( + /import snippets\//g, + 'import /etc/caddy/sites/legacy-snippets/', + ), + ), + ); + }); + + test("main's snippets get the same variables", () => { + const headers = readFileSync(join(MAIN, 'caddy', 'snippets', 'SecurityHeaders'), 'utf8'); + assert.match( + fillEnvironment(headers, { ...VALUES, adminDomain: 'admin.example.com' }), + /frame-ancestors 'self' admin\.example\.com/, + ); + assert.match(fillEnvironment(headers, VALUES), /frame-ancestors 'self' "/); + const activitypub = readFileSync(join(MAIN, 'caddy', 'snippets', 'ActivityPub'), 'utf8'); + assert.match( + fillEnvironment(activitypub, { ...VALUES, activitypub: 'activitypub:8080' }), + /reverse_proxy activitypub:8080/, + ); + }); + + test('custom routes are kept as written', () => { + const custom = `${EXAMPLE}\nstatus.example.com {\n\theader X-Host {env.DOMAIN}\n\theader X-Other {$OTHER}\n\treverse_proxy 172.17.0.1:9000\n}\n`; + const { site } = carry(custom); + assert.match( + site, + /^status\.example\.com \{\n\theader X-Host example\.com\n\theader X-Other \{\$OTHER\}\n\treverse_proxy 172\.17\.0\.1:9000\n\}$/m, + ); + }); + + test('a leading global options block moves to caddy/global, one level shallower', () => { + const { site, global } = carry( + `# mine\n{\n\temail ops@example.com\n\tdebug\n}\n\n${EXAMPLE}`, + ); + assert.equal( + global, + '# Global options carried over from caddy/Caddyfile.\nemail ops@example.com\ndebug\n', + ); + assert.doesNotMatch(code(site), /email ops/); + assert.match(code(site), /^example\.com \{$/m); + }); +}); + +describe('the .env', () => { + const OLD = [ + 'COMPOSE_PROFILES=analytics', + 'DOMAIN=example.com', + 'ADMIN_DOMAIN=', + 'HTTP_PORT=80', + 'DATABASE_ROOT_PASSWORD=root', + 'DATABASE_PASSWORD=app', + 'TINYBIRD_ADMIN_TOKEN=p.token', + 'TINYBIRD_SYNC_AUTH=shared', + 'mail__transport=SMTP', + 'mail__options__auth__pass="p$$ss"', + 'mail__from="\'Acme\' "', + 'url=https://ignored.example.com', + 'database__connection__host=elsewhere', + 'TZ=Europe/London', + 'GHOST_VERSION=6-alpine', + '', + ].join('\n'); + const OPERATOR = new Set([ + 'DOMAIN', + 'ADMIN_DOMAIN', + 'HTTP_PORT', + 'DATABASE_ROOT_PASSWORD', + 'DATABASE_PASSWORD', + 'TINYBIRD_ADMIN_TOKEN', + 'TINYBIRD_SYNC_AUTH', + 'GHOST_VERSION', + ]); + const split = (interpolated: string[] = []) => + splitLegacyEnv({ + legacy: readLegacyEnv(OLD), + generated: new Set([ + 'COMPOSE_PROFILES', + 'HTTP_PORT', + 'DATABASE_ROOT_PASSWORD', + 'DATABASE_PASSWORD', + 'GHOST_VERSION', + ]), + isOperatorKey: (key) => key.startsWith('COMPOSE_') || OPERATOR.has(key), + isInterpolated: (key) => interpolated.includes(key), + container: new Set(['url', 'admin__url', 'NODE_ENV']), + received: { mail__options__auth__pass: 'p$ss', mail__transport: 'SMTP' }, + }); + + test("Ghost's configuration moves to ghost.env, operator settings stay", () => { + const { operator, ghost, dropped } = split(); + assert.deepEqual(operator, [ + ['TINYBIRD_ADMIN_TOKEN', 'p.token'], + ['TINYBIRD_SYNC_AUTH', 'shared'], + ]); + assert.deepEqual(ghost, [ + ['mail__transport', 'SMTP'], + ['mail__options__auth__pass', 'p$ss'], + ['mail__from', "'Acme' "], + // Everything in .env reached Ghost on main, so a setting of the + // container's own goes with it. + ['TZ', 'Europe/London'], + ]); + assert.deepEqual( + dropped.map(({ key }) => key), + ['DOMAIN', 'ADMIN_DOMAIN', 'url', 'database__connection__host'], + ); + }); + + test('DOMAIN stays when an override still uses it', () => { + const { operator, dropped } = split(['DOMAIN']); + assert.deepEqual(operator[0], ['DOMAIN', 'example.com']); + assert.ok(!dropped.some(({ key }) => key === 'DOMAIN')); + }); + + test('ghost.env gets what Ghost received, which Compose interpolated', () => { + const { ghost } = split(); + assert.deepEqual( + ghost.find(([key]) => key === 'mail__options__auth__pass'), + ['mail__options__auth__pass', 'p$ss'], + ); + }); + + test('an operator value Compose would interpolate is refused', () => { + assert.throws( + () => refuseInterpolated('DATABASE_PASSWORD=pa$word\n', new Set(['DATABASE_PASSWORD'])), + /unescaped \$ \(DATABASE_PASSWORD\)/, + ); + // Ghost's own are carried as Ghost received them. + refuseInterpolated('mail__options__auth__pass=pa$word\n', new Set()); + }); + + test('a value spanning lines is refused rather than dropped', () => { + assert.throws(() => readLegacyEnv('KEY="one\ntwo"\n'), /KEY/); + }); +}); diff --git a/manager/test/migrate-main.test.ts b/manager/test/migrate-main.test.ts new file mode 100644 index 00000000..a95f7384 --- /dev/null +++ b/manager/test/migrate-main.test.ts @@ -0,0 +1,462 @@ +// `self-update` in an installation of the released main layout: migration +// 0001-compose-profiles, against a scripted daemon, Compose and git. What it +// does on a real host is tests/e2e/migrate-main.sh; this is what it decides. +import assert from 'node:assert/strict'; +import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { basename, join } from 'node:path'; +import { afterEach, beforeEach, describe, test } from 'node:test'; +import * as env from '../src/env.ts'; +import { acquireLock } from '../src/lock.ts'; +import { failed, harness, json, ok, type Harness, type ProgramResult } from './helpers.ts'; +import { + imageApi, + imageStack, + REFERENCE, + releasedMain, + REPO, + resolvedProject, + scriptSite, + writeSiteData, + type ScriptedSite, +} from './site.ts'; + +const MAIN = releasedMain(); +/** The Ghost-CLI-layout image main's site runs. */ +const OLD_GHOST = `sha256:${'5'.repeat(64)}`; +const CADDY = `caddy:2.10.2-alpine@sha256:${'6'.repeat(64)}`; +const DB = `mysql:8.0.44@sha256:${'4'.repeat(64)}`; + +const OLD_ENV = [ + '# Use the below flags to enable the Analytics or ActivityPub containers as well', + 'COMPOSE_PROFILES=activitypub', + 'DOMAIN=example.com', + 'HTTP_PORT=80', + 'HTTPS_PORT=443', + 'DATABASE_ROOT_PASSWORD=reallysecurerootpassword', + 'DATABASE_PASSWORD=ghostpassword', + 'ACTIVITYPUB_TARGET=activitypub:8080', + 'mail__transport=SMTP', + 'mail__options__host=smtp.example.com', + 'mail__options__auth__pass="pa$$word"', + 'mail__from="\'Acme Support\' "', + 'labs__publicAPI=true', + 'UPLOAD_LOCATION=./data/ghost', + 'MYSQL_DATA_LOCATION=./data/mysql', + '', +].join('\n'); + +let h: Harness; +let site: ScriptedSite; +let project: string; +/** Files git says the operator changed, and files only local commits changed. */ +let edited: string[]; +let committed: string[]; +/** What `caddy validate` answers. */ +let caddyValidates: { status: number; stderr?: string }; +const validated: string[] = []; +/** A mount the operator's override adds to the ghost service, in either layout. */ +let overrideMount: { type: string; source: string; target: string } | null; + +const readSite = (file: string) => readFileSync(join(h.dir, file), 'utf8'); +const envOf = (file: string) => env.toRecord(readSite(file)); +const migrate = (...args: string[]) => h.run('self-update', ...args); + +/** Every file outside the data, to compare before and after. */ +function files(dir = h.dir, prefix = ''): Record { + const found: Record = {}; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (prefix === '' && ['data', 'backups'].includes(entry.name)) { + continue; + } + const path = join(dir, entry.name); + if (entry.isDirectory()) { + Object.assign(found, files(path, `${prefix}${entry.name}/`)); + } else { + found[`${prefix}${entry.name}`] = readFileSync(path, 'utf8'); + } + } + return found; +} + +/** `docker compose config`: main's layout with the site's `.env`, or this one's with a staged one. */ +function config(_args: string[], envFile?: string): ProgramResult { + const values = env.toRecord(readFileSync(envFile ?? join(h.dir, '.env'), 'utf8')); + const legacy = values.URL === undefined; + // What the override requires, as Compose interpolates `${NAME:?...}`. + const override = existsSync(join(h.dir, 'compose.override.yml')) + ? readSite('compose.override.yml') + : ''; + for (const [, name] of override.matchAll(/\$\{([A-Za-z_]\w*):\?/g)) { + if (!values[name!]) { + return failed(1, `required variable ${name} is missing a value`); + } + } + const environment: Record = legacy + ? // main's env_file is the whole of .env. + { ...values, url: `https://${values.DOMAIN}`, NODE_ENV: 'production' } + : { + ...env.toRecord(existsSync(join(h.dir, 'ghost.env')) ? readSite('ghost.env') : ''), + NODE_ENV: 'production', + url: values.URL!, + admin__url: values.ADMIN_URL ?? '', + server__host: '0.0.0.0', + server__port: '2368', + paths__contentPath: values.GHOST_CONTENT_PATH ?? '', + database__client: 'mysql', + }; + const resolved = JSON.parse( + resolvedProject( + h.dir, + { ghost: legacy ? 'ghost:6-alpine' : values.GHOST_IMAGE_REF!, db: DB, caddy: CADDY }, + {}, + envFile, + ), + ); + resolved.services.ghost.environment = Object.fromEntries( + Object.entries(environment).map(([key, value]) => [key, value.replaceAll('$', '$$')]), + ); + if (legacy) { + resolved.services.ghost.volumes[0].target = '/var/lib/ghost/content'; + } + if (overrideMount !== null) { + resolved.services.ghost.volumes.push(overrideMount); + } + return ok(JSON.stringify(resolved)); +} + +beforeEach(() => { + h = harness(); + imageStack(h); + h.env.GD_VERSION_FILE = join(h.dir, '..', `${basename(h.dir)}-version.json`); + writeFileSync( + h.env.GD_VERSION_FILE, + JSON.stringify({ version: 'v0.2.0', commit: 'c'.repeat(40) }), + ); + h.env.GD_CHANNEL = 'stable'; + + // A clone of main, set up as its README said. + cpSync(MAIN, h.dir, { recursive: true }); + writeFileSync(join(h.dir, '.env'), OLD_ENV); + cpSync(join(MAIN, 'caddy', 'Caddyfile.example'), join(h.dir, 'caddy', 'Caddyfile')); + writeSiteData(h.dir); + project = basename(h.dir); + edited = []; + committed = []; + caddyValidates = { status: 0 }; + overrideMount = null; + validated.length = 0; + + const images = imageApi({ ghost: { '6.67.0-next-alpine': '6.67.0' } }); + h.daemon.api = (request) => { + if ( + request.method === 'GET' && + decodeURIComponent(request.path) === `/images/${OLD_GHOST}/json` + ) { + return json(200, { + Id: OLD_GHOST, + RepoDigests: [`ghost@sha256:${'7'.repeat(64)}`], + Config: { + Env: [ + 'GHOST_VERSION=6.67.0', + 'GHOST_INSTALL=/var/lib/ghost', + 'GHOST_CONTENT=/var/lib/ghost/content', + 'GHOST_CLI_INSTALL=/usr/local/lib/ghost-cli', + ], + }, + }); + } + return images(request); + }; + const container = (service: string, image: string, imageId: string) => ({ + Id: `${service}-id`, + Names: [`/${project}-${service}-1`], + Image: image, + ImageID: imageId, + State: 'running', + Status: 'Up', + Labels: { + 'com.docker.compose.project': project, + 'com.docker.compose.project.working_dir': h.dir, + 'com.docker.compose.service': service, + }, + }); + h.daemon.containers = [ + container('ghost', 'ghost:6-alpine', OLD_GHOST), + container('db', DB, `sha256:${'4'.repeat(64)}`), + container('caddy', CADDY, `sha256:${'6'.repeat(64)}`), + ]; + h.daemon.gitRun = (args) => { + const command = args[args.indexOf(h.dir) + 1]; + const paths = args.slice(args.indexOf('--') + 1); + switch (command) { + case 'rev-parse': + return ok(`${h.dir}\n`); + case 'ls-files': + return ok(paths.filter((path) => existsSync(join(MAIN, path))).join('\n')); + case 'status': + return ok(edited.map((file) => ` M ${file}\n`).join('')); + case 'for-each-ref': + return ok('refs/remotes/origin/main\n'); + case 'log': + return ok(committed.join('\n')); + default: + return failed(1, `unexpected: git ${args.join(' ')}`); + } + }; + + site = scriptSite(h, config, { ghost: REFERENCE, db: DB, caddy: CADDY }); + site.running.add('ghost').add('db').add('caddy'); + site.onUp = () => site.running.add('caddy'); + const scripted = h.daemon.run!; + h.daemon.run = (spec) => { + if (spec.entrypoint[0] === 'caddy') { + const caddy = spec.binds[0]!.split(':')[0]!; + validated.push(readFileSync(join(caddy, 'sites', 'site.caddy'), 'utf8')); + return caddyValidates; + } + return scripted(spec); + }; +}); +afterEach(() => h.cleanup()); + +describe('migrating the released main layout', () => { + test('moves the site onto this layout, keeping its project, credentials and Ghost', async () => { + const result = await migrate(); + assert.equal(result.code, 0, result.stderr); + assert.match(result.stdout, /migration 0001-compose-profiles/); + + const settings = envOf('.env'); + assert.equal(settings.COMPOSE_PROFILES, 'production,activitypub'); + assert.equal(settings.SITE_MODE, 'production'); + assert.equal(settings.COMPOSE_PROJECT_NAME, project); + assert.equal(settings.PROJECT_DIR, h.dir); + assert.equal(settings.URL, 'https://example.com'); + assert.equal(settings.GHOST_IMAGE_REF, REFERENCE); + assert.equal(settings.GHOST_VERSION, '6.67.0-next-alpine'); + assert.equal(settings.DATABASE_PASSWORD, 'ghostpassword'); + assert.equal(settings.DATABASE_ROOT_PASSWORD, 'reallysecurerootpassword'); + for (const key of ['DOMAIN', 'ACTIVITYPUB_TARGET', 'mail__transport']) { + assert.equal(settings[key], undefined, key); + } + + const ghost = envOf('ghost.env'); + assert.equal(ghost.mail__transport, 'SMTP'); + assert.equal(ghost.mail__options__auth__pass, 'pa$word'); + assert.equal(ghost.labs__publicAPI, 'true'); + assert.equal(ghost.DATABASE_PASSWORD, undefined); + + const routes = readSite('caddy/sites/site.caddy'); + assert.match(routes, /^example\.com \{$/m); + assert.match(routes, /^\timport \/etc\/caddy\/sites\/legacy-snippets\/ActivityPub$/m); + assert.match(routes, /reverse_proxy ghost:2368$/m); + assert.match( + readSite('caddy/sites/legacy-snippets/ActivityPub'), + /reverse_proxy activitypub:8080$/m, + ); + assert.equal( + readSite('caddy/Caddyfile.local'), + readFileSync(join(MAIN, 'caddy', 'Caddyfile.example'), 'utf8'), + ); + assert.equal( + readSite('caddy/Caddyfile'), + readFileSync(join(REPO, 'caddy', 'Caddyfile'), 'utf8'), + ); + assert.equal(readSite('compose.yml'), readFileSync(join(REPO, 'compose.yml'), 'utf8')); + assert.equal(validated.length, 1); + + const metadata = JSON.parse(readSite('.ghost-docker.json')); + assert.equal(metadata.source, 'image'); + assert.equal(metadata.stack.version, 'v0.2.0'); + assert.equal(metadata.site.project, project); + assert.equal(metadata.ghost.version, '6.67.0'); + assert.deepEqual(metadata.profiles, ['production', 'activitypub']); + assert.ok('compose.yml' in metadata.payload && 'ghost-docker' in metadata.payload); + assert.match(readSite('ghost-docker'), /^readonly GD_PINNED_IMAGE=".+"$/m); + + assert.equal(readdirSync(join(h.dir, 'backups')).length, 1); + assert.ok(!existsSync(join(h.dir, '.ghost-docker-update'))); + assert.ok(!existsSync(join(h.dir, '.ghost-docker.lock'))); + assert.deepEqual([...site.running].sort(), ['caddy', 'db', 'ghost']); + }); + + test('ActivityPub stays where main sent it: the hosted service unless the target said otherwise', async () => { + writeFileSync( + join(h.dir, '.env'), + OLD_ENV.replace('ACTIVITYPUB_TARGET=activitypub:8080\n', ''), + ); + const result = await migrate(); + assert.equal(result.code, 0, result.stderr); + assert.match( + readSite('caddy/sites/legacy-snippets/ActivityPub'), + /reverse_proxy https:\/\/ap\.ghost\.org$/m, + ); + assert.equal(envOf('.env').COMPOSE_PROFILES, 'production,activitypub'); + }); + + test('a second run is an ordinary self-update, and finds nothing to do', async () => { + assert.equal((await migrate()).code, 0); + const after = files(); + const again = await migrate(); + assert.equal(again.code, 0, again.stderr); + assert.match(again.stdout, /already runs v0\.2\.0/); + assert.deepEqual(files(), after); + }); + + test('--check says what it would do and changes nothing', async () => { + const before = files(); + const result = await migrate('--check'); + assert.equal(result.code, 0, result.stderr); + assert.match(result.stdout, /would move this site onto v0\.2\.0/); + assert.match(result.stdout, /DOMAIN not carried/); + assert.deepEqual(files(), before); + assert.equal(site.compose.filter((args) => args[0] === 'stop').length, 0); + }); + + test('an edited stack file stops it before anything changes', async () => { + edited = ['compose.yml']; + committed = ['caddy/snippets/Logging']; + const before = files(); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /caddy\/snippets\/Logging\n\s+compose\.yml/); + assert.match(result.stderr, /Nothing has been changed/); + assert.deepEqual(files(), before); + }); + + test('Ghost below the first next image stops it, saying to upgrade Ghost first', async () => { + const api = h.daemon.api!; + h.daemon.api = (request) => + decodeURIComponent(request.path) === `/images/${OLD_GHOST}/json` + ? json(200, { + Id: OLD_GHOST, + RepoDigests: [`ghost@sha256:${'7'.repeat(64)}`], + Config: { Env: ['GHOST_VERSION=6.40.0'] }, + }) + : api(request); + const before = files(); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /runs Ghost 6\.40\.0.*start at Ghost 6\.61\.0/s); + assert.deepEqual(files(), before); + }); + + test('routes Caddy will not load stop it before anything changes', async () => { + caddyValidates = { + status: 1, + stderr: 'Error: adapting config: unknown directive: frobnicate', + }; + const before = files(); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /unknown directive: frobnicate/); + assert.deepEqual(files(), before); + }); + + test('a failure before startup puts the files back and starts Ghost again', async () => { + const before = files(); + // The checked backup fails: its dump does not load. + const scripted = h.daemon.run!; + h.daemon.run = (spec) => + spec.entrypoint[0] === 'sh' + ? { status: 4, stderr: 'the dump of ghost does not load' } + : scripted(spec); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /Restored: the site is on the released main layout again/); + assert.deepEqual(files(), before); + assert.ok(site.running.has('ghost')); + assert.equal( + site.compose.filter((args) => args[0] === 'up' && !args.includes('--no-recreate')) + .length, + 0, + ); + }); + + test('a failure at startup stops the site, puts the files back and leaves the data', async () => { + const before = files(); + site.ups.push(failed(1, 'container ghost is unhealthy')); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /The site needs you/); + assert.match(result.stderr, /Nothing was loaded over them/); + assert.match(result.stderr, /docker compose up -d/); + assert.match(result.stderr, /restore --yes backups\//); + // The snapshot stays for the operator; everything else is main's again. + const after = files(); + assert.deepEqual( + Object.fromEntries( + Object.entries(after).filter(([path]) => !path.startsWith('.ghost-docker-update/')), + ), + before, + ); + assert.equal( + readFileSync(join(h.dir, 'data', 'ghost', 'images', 'photo.jpg'), 'utf8'), + 'jpeg', + ); + }); + + test('a second run waits for no one: another holding the lock is refused, its snapshot kept', async () => { + // Another migration, part-way through: its lock and its snapshot. + const other = acquireLock( + h.dir, + 'migration 0001-compose-profiles from the released main layout', + ); + mkdirSync(join(h.dir, '.ghost-docker-update', 'files'), { recursive: true }); + writeFileSync(join(h.dir, '.ghost-docker-update', 'files', '.env'), OLD_ENV); + try { + for (const args of [[], ['--check']]) { + const result = await migrate(...args); + assert.equal(result.code, 1); + // Refused by the lock, before it looks at, stages or removes anything. + assert.match( + result.stderr, + /\.ghost-docker\.lock is held by migration 0001-compose-profiles/, + ); + assert.ok(!h.calls.some((call) => call[0] === 'git'), 'it began preparing'); + assert.equal( + readSite('.ghost-docker-update/files/.env'), + OLD_ENV, + 'the other run’s snapshot was touched', + ); + } + } finally { + other.release(); + } + }); + + test("an override mounting into main's Ghost install is refused: this layout's Ghost never looks there", async () => { + overrideMount = { + type: 'bind', + source: join(h.dir, 'images'), + target: '/var/lib/ghost/content/images', + }; + const before = files(); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match( + result.stderr, + /at \/var\/lib\/ghost\/content\/images in the\n\s+ghost service/, + ); + assert.match(result.stderr, /Nothing has been changed/); + assert.deepEqual(files(), before); + }); + + test('an override requiring a setting .env has resolves with the new layout too', async () => { + writeFileSync( + join(h.dir, 'compose.override.yml'), + 'services:\n ghost:\n environment:\n mail__options__host: ${SMTP_HOST:?SMTP_HOST is required}\n', + ); + writeFileSync(join(h.dir, '.env'), `${OLD_ENV}SMTP_HOST=smtp.example.com\n`); + const result = await migrate(); + assert.equal(result.code, 0, result.stderr); + assert.equal(envOf('.env').SMTP_HOST, 'smtp.example.com'); + assert.equal(envOf('ghost.env').SMTP_HOST, undefined); + }); + + test('a site that is not running is refused: its Ghost version is unknown', async () => { + h.daemon.containers = []; + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /must be running/); + }); +}); diff --git a/manager/test/site.ts b/manager/test/site.ts index f1ca47d7..45d1afae 100644 --- a/manager/test/site.ts +++ b/manager/test/site.ts @@ -1,6 +1,7 @@ // A site directory for tests: the repository's own compose.yml and examples, // a `.env` written through the real encoder, and a scripted // `docker compose config` that answers what Compose would. +import { execFileSync } from 'node:child_process'; import { copyFileSync, cpSync, @@ -29,6 +30,42 @@ import { export const REPO = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +/** + * The released main layout's commit: `origin/main` until `next-docker` is + * merged into it, then pinned to main's last commit before that merge (S12 in + * docs/ghost-cli-replacement.md). Read from git, so it never drifts from main. + */ +export const RELEASED_MAIN = process.env.GD_RELEASED_MAIN ?? 'origin/main'; + +let releasedMainDir: string | undefined; + +/** + * The released main layout's files, checked out once per test process into a + * directory of their own, removed when the process exits. + */ +export function releasedMain(): string { + if (releasedMainDir !== undefined) { + return releasedMainDir; + } + let archive: Buffer; + try { + archive = execFileSync('git', ['-C', REPO, 'archive', '--format=tar', RELEASED_MAIN], { + stdio: ['ignore', 'pipe', 'pipe'], + maxBuffer: 256 * 1024 * 1024, + }); + } catch (error) { + throw new Error( + `the released main layout (${RELEASED_MAIN}) is not in this repository: ` + + `git fetch origin main, or set GD_RELEASED_MAIN. ${(error as Error).message}`, + ); + } + const dir = realpathSync(mkdtempSync(join(tmpdir(), 'gd-released-main-'))); + execFileSync('tar', ['-x', '-C', dir], { input: archive }); + process.on('exit', () => rmSync(dir, { recursive: true, force: true })); + releasedMainDir = dir; + return dir; +} + /** A bundle manifest, loosely typed so tests can break it. */ export type Manifest = Record; @@ -322,7 +359,7 @@ export function writeSiteData(dir: string): void { */ export function scriptSite( h: Harness, - config: (args: string[]) => ProgramResult | undefined, + config: (args: string[], envFile?: string) => ProgramResult | undefined, images: Record = { ghost: REFERENCE, db: `mysql:8.0.44@sha256:${'4'.repeat(64)}`, @@ -362,11 +399,11 @@ export function scriptSite( } return undefined; }; - h.daemon.composeRun = (args, _env, input) => { + h.daemon.composeRun = (args, _env, input, _dir, envFile) => { site.compose.push(args); switch (args[0]) { case 'config': { - const answer = config(args); + const answer = config(args, envFile); if (answer === undefined || answer.exitCode !== 0) { return answer; } diff --git a/tests/e2e/migrate-main.sh b/tests/e2e/migrate-main.sh new file mode 100755 index 00000000..0006f461 --- /dev/null +++ b/tests/e2e/migrate-main.sh @@ -0,0 +1,289 @@ +#!/usr/bin/env bash +# Migration 0001-compose-profiles, against real containers: an installation +# of the released main layout moved onto this one by the served launcher. +# +# tests/e2e/migrate-main.sh +# +# The installation is made as main's README made one: a git clone of the +# repository at GD_RELEASED_MAIN (origin/main until next-docker is merged +# into it; then main's last commit before that merge, S12 in +# docs/ghost-cli-replacement.md), `.env` and `caddy/Caddyfile` copied from the examples and +# edited, ActivityPub enabled, a custom route, a global options block, and a +# compose.override.yml. It runs the official `ghost:6-alpine` image, whose +# layout this release does not run. Then: +# +# - an edited stack file, and a release image that cannot be pulled, stop +# the migration before anything changes; +# - Ghost never becoming healthy on the new layout stops the site, puts +# main's files back, and the site starts again on them with its data; +# - the migration through the served launcher, piped as from curl, keeps +# the project, its volumes, credentials, data, Ghost's version, the +# routes and the override, and the site answers through Caddy; +# - running it again is an ordinary self-update with nothing to do. +# +# It pulls images, starts containers and publishes Caddy on two loopback +# ports. Caddy's own CA stands in for a real one (`local_certs`), so nothing +# asks an ACME server. Exits non-zero at the first check that fails. +set -euo pipefail + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd -P) +REGISTRY=ghcr.io/tryghost/ghost-docker +# A version no real release has, so a published image is never mistaken for it. +RELEASE=v0.0.2-beta.1 +IMAGE=$REGISTRY:$RELEASE +DOMAIN=ghost-e2e.test +HTTP_PORT=18080 +HTTPS_PORT=18443 +MAIN_REPOSITORY=${GD_MAIN_REPOSITORY:-https://github.com/TryGhost/ghost-docker.git} +RELEASED_MAIN=${GD_RELEASED_MAIN:-origin/main} + +# shellcheck source=/dev/null +source "$ROOT/tests/e2e/skip.sh" +require_docker +for tool in git curl; do + command -v "$tool" >/dev/null || { + printf 'migrate-main.sh needs %s\n' "$tool" >&2 + exit 1 + } +done + +WORK=$(mktemp -d "${TMPDIR:-/tmp}/ghost-docker-migrate-e2e.XXXXXXXX") +WORK=$(CDPATH='' cd -- "$WORK" && pwd -P) +S=$WORK/sites/main-site +CURRENT=setup +OUT="" +RC=0 + +cleanup() { + local rc=$? + set +e + [[ -f $S/compose.yml ]] && docker compose --project-directory "$S" -f "$S/compose.yml" \ + --profile '*' down --volumes --remove-orphans --timeout 5 >/dev/null 2>&1 + docker run --rm --user 0 --entrypoint rm -v "$WORK:/work" "$IMAGE" -rf /work/sites >/dev/null 2>&1 + docker rmi "$IMAGE" >/dev/null 2>&1 + case $WORK in */ghost-docker-migrate-e2e.*) rm -rf -- "$WORK" ;; esac + exit "$rc" +} +trap cleanup EXIT + +step() { + CURRENT=$1 + printf '\n== %s\n' "$1" +} +ok() { printf ' ok %s\n' "$1"; } +fail() { + printf '\nFAILED in "%s": %s\n' "$CURRENT" "$1" >&2 + [[ -z ${2:-} ]] || printf -- '--- output ---\n%s\n--------------\n' "$2" >&2 + exit 1 +} +run() { + set +e + OUT=$("$@" 2>&1 /dev/null | grep -q 'Ghost booted'; do + ((waited < 300)) || fail "Ghost did not boot" "$(compose_in logs --tail 40 ghost 2>&1)" + sleep 5 + waited=$((waited + 5)) + done +} +root_sql() { + compose_in exec -T -e MYSQL_PWD=e2e-root-password db mysql -uroot -N -B ghost -e "$1" +} +# https PATH -- through Caddy on the host's port, as a browser would ask. +https() { + curl --silent --insecure --noproxy '*' --max-time 20 --header "Host: $DOMAIN" \ + --resolve "$DOMAIN:$HTTPS_PORT:127.0.0.1" "https://$DOMAIN:$HTTPS_PORT$1" || true +} + +# --- The release, and main's installation ----------------------------------------- + +step "Build the release" +run docker build --quiet --file "$ROOT/manager/Dockerfile" --build-arg "GD_VERSION=$RELEASE" \ + --build-arg "GD_COMMIT=$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || printf '')" --tag "$IMAGE" "$ROOT" +expect_status 0 +ok "$IMAGE" + +step "An installation of the released main layout, as its README made one" +mkdir -p "$WORK/sites" +run git clone --quiet "$MAIN_REPOSITORY" "$S" +expect_status 0 +run git -C "$S" checkout --quiet --detach "$RELEASED_MAIN" +expect_status 0 +cp "$S/.env.example" "$S/.env" +sed -i.bak \ + -e "s/^# COMPOSE_PROFILES=.*/COMPOSE_PROFILES=activitypub/" \ + -e "s/^DOMAIN=.*/DOMAIN=$DOMAIN/" \ + -e "s/^HTTP_PORT=.*/HTTP_PORT=$HTTP_PORT/" \ + -e "s/^HTTPS_PORT=.*/HTTPS_PORT=$HTTPS_PORT/" \ + -e "s/^DATABASE_ROOT_PASSWORD=.*/DATABASE_ROOT_PASSWORD=e2e-root-password/" \ + -e "s/^DATABASE_PASSWORD=.*/DATABASE_PASSWORD=e2e-app-pa\$\$word/" \ + -e "s/^# ACTIVITYPUB_TARGET=.*/ACTIVITYPUB_TARGET=activitypub:8080/" \ + -e "s/^mail__options__auth__pass=.*/mail__options__auth__pass=e2e-smtp-secret/" \ + "$S/.env" +rm "$S/.env.bak" +printf 'labs__publicAPI=true\n' >>"$S/.env" +# The example, with Caddy's own CA, and a route of the operator's own. +{ + printf '{\n\tlocal_certs\n}\n\n' + sed 's|^\t# Default proxy everything else to Ghost|\thandle /e2e-custom {\n\t\trespond "custom route kept"\n\t}\n\n\t# Default proxy everything else to Ghost|' \ + "$S/caddy/Caddyfile.example" +} >"$S/caddy/Caddyfile" +cat >"$S/compose.override.yml" <<'EOF' +services: + ghost: + environment: + e2e__override: kept +EOF +run compose_in up -d +expect_status 0 +wait_for_ghost +project=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$(ghost_id)") +version=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$(ghost_id)" | sed -n 's/^GHOST_VERSION=//p') +root_sql 'CREATE TABLE e2e_marker (note varchar(64)); INSERT INTO e2e_marker VALUES ("kept")' +compose_in exec -T ghost sh -c 'printf kept >/var/lib/ghost/content/images/e2e-marker.txt' +[[ $(https /e2e-custom) == 'custom route kept' ]] || fail "main's custom route does not answer" "$(https /e2e-custom)" +docker volume inspect "${project}_caddy_data" >/dev/null || fail "no ${project}_caddy_data volume" +ok "$(git -C "$S" rev-parse --short HEAD) of main: project $project, Ghost $version on ghost:6-alpine, with ActivityPub, a custom route and an override" + +# --- Refused before anything changes ------------------------------------------------ + +step "An edited stack file stops it before anything changes" +printf '# mine\n' >>"$S/compose.yml" +before=$(fingerprint) +ghost_before=$(ghost_id) +served self-update +expect_status 1 +expect_output 'these files of the stack were changed here' +expect_output 'compose\.yml' +expect_output 'Nothing has been changed' +[[ $(fingerprint) == "$before" && $(ghost_id) == "$ghost_before" ]] || fail "a refused migration changed the site" +git -C "$S" checkout --quiet -- compose.yml +ok "refused, naming compose.yml" + +step "An image the release cannot pull stops it before anything changes" +cp "$S/compose.override.yml" "$WORK/override.yml" +cat >>"$S/compose.override.yml" <<'EOF' + e2e-unpullable: + image: ghcr.io/tryghost/ghost-docker-e2e-does-not-exist:1 + profiles: [production] +EOF +before=$(fingerprint) +served self-update +expect_status 1 +expect_output "images could not be pulled" +expect_output 'Nothing has been changed' +[[ $(fingerprint) == "$before" && $(ghost_id) == "$ghost_before" ]] || fail "a refused migration changed the site" +cp "$WORK/override.yml" "$S/compose.override.yml" +ok "refused while the site kept running" + +step "--check says what it would do and changes nothing" +before=$(fingerprint) +served self-update --check +expect_status 0 +expect_output "would move this site onto $RELEASE" +expect_output "Ghost +$version" +[[ $(fingerprint) == "$before" && $(ghost_id) == "$ghost_before" ]] || fail "--check changed the site" +ok "nothing changed" + +# --- Failing after startup --------------------------------------------------------- + +step "Ghost never healthy on the new layout: stopped, main's files back, data kept" +cat >>"$S/compose.override.yml" <<'EOF' + healthcheck: + test: [CMD, "false"] + interval: 2s + start_period: 0s + start_interval: 2s + retries: 1 +EOF +before=$(fingerprint) +served self-update +expect_status 1 +expect_output 'The site needs you' +expect_output 'Nothing was loaded over them' +expect_output 'docker compose up -d' +[[ $(fingerprint) == "$before" ]] || fail "main's files were not put back" "$(diff <(printf '%s\n' "$before") <(fingerprint) || true)" +[[ -z $(compose_in ps -q) ]] || fail "the migrated site was left running" "$(compose_in ps)" +ok "stopped, with main's files back" + +# The operator's part: the snapshot and the backup are no longer needed. +rm -rf "$S/.ghost-docker-update" "$S/backups" +cp "$WORK/override.yml" "$S/compose.override.yml" +run compose_in up -d +expect_status 0 +wait_for_ghost +[[ $(root_sql 'SELECT note FROM e2e_marker') == kept ]] || fail "the marker row is gone" +[[ $(https /e2e-custom) == 'custom route kept' ]] || fail "main's site does not answer again" +ok "main's layout starts again on its data" + +# --- The migration -------------------------------------------------------------------- + +step "The served launcher migrates it" +served self-update +expect_status 0 +expect_output "Moved from the released main layout to $RELEASE" +expect_output 'with a certificate from Caddy Local Authority' +setting() { "$S/ghost-docker" --dir "$S" config get "$1" 2>/dev/null; } +[[ $(setting COMPOSE_PROFILES) == production,activitypub ]] || fail "COMPOSE_PROFILES is $(setting COMPOSE_PROFILES)" +[[ $(setting COMPOSE_PROJECT_NAME) == "$project" ]] || fail "the project is $(setting COMPOSE_PROJECT_NAME)" +# shellcheck disable=SC2016 # a literal $, as Compose read it from main's .env +[[ $(setting DATABASE_PASSWORD) == 'e2e-app-pa$word' ]] || fail "the database password changed" +[[ $(setting URL) == "https://$DOMAIN" ]] || fail "URL is $(setting URL)" +[[ $("$S/ghost-docker" --dir "$S" config get ghost.env mail__options__auth__pass 2>/dev/null) == e2e-smtp-secret ]] || + fail "the SMTP password did not move to ghost.env" +grep -q '^DOMAIN=' "$S/.env" && fail ".env still has DOMAIN" +grep -q 'local_certs' "$S/caddy/global/legacy.caddy" || fail "the global options were not carried" +grep -q 'import /etc/caddy/sites/legacy-snippets/ActivityPub' "$S/caddy/sites/site.caddy" || + fail "the routes do not import main's snippets" "$(cat "$S/caddy/sites/site.caddy")" +grep -q 'reverse_proxy activitypub:8080' "$S/caddy/sites/legacy-snippets/ActivityPub" || + fail "ActivityPub is not where main sent it" "$(cat "$S/caddy/sites/legacy-snippets/ActivityPub")" +cmp -s "$S/caddy/Caddyfile" "$ROOT/caddy/Caddyfile" || fail "caddy/Caddyfile is not the release's" +[[ -f $S/caddy/Caddyfile.local ]] || fail "the old Caddyfile was not kept" +running=$(docker inspect -f '{{.Config.Image}}' "$(ghost_id)") +[[ $running == "$(setting GHOST_IMAGE_REF)" && $running == ghost@sha256:* ]] || fail "Ghost runs $running" +now=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$(ghost_id)") +grep -qx "GHOST_VERSION=$version" <<<"$now" || fail "Ghost's version changed" "$now" +grep -qx 'e2e__override=kept' <<<"$now" || fail "the override does not reach Ghost" "$now" +grep -q 'DATABASE_ROOT_PASSWORD' <<<"$now" && fail "Ghost receives .env's root password" +[[ $(root_sql 'SELECT note FROM e2e_marker') == kept ]] || fail "the marker row is gone" +[[ $(compose_in exec -T ghost cat /home/ghost/content/images/e2e-marker.txt) == kept ]] || fail "the content is gone" +[[ $(https /e2e-custom) == 'custom route kept' ]] || fail "the custom route does not answer" "$(https /e2e-custom)" +[[ $(https /ghost/api/admin/site/) == *"\"url\":\"https://$DOMAIN/\""* ]] || fail "Ghost does not answer through Caddy" +docker volume inspect "${project}_caddy_data" >/dev/null || fail "Caddy's volume is not the site's" +[[ $(find "$S/backups" -mindepth 1 -maxdepth 1 -type d | wc -l) -eq 1 ]] || fail "there is not one backup" +[[ ! -e $S/.ghost-docker-update && ! -e $S/.ghost-docker.lock ]] || fail "the migration left its snapshot or lock" +ok "Ghost $version on its next image, project, volumes, data, routes and override kept" + +run "$S/ghost-docker" --dir "$S" check +expect_status 0 +ok "check passes" + +step "Running it again is an ordinary self-update, with nothing to do" +before=$(fingerprint) +served self-update +expect_status 0 +expect_output "already runs $RELEASE" +[[ $(fingerprint) == "$before" ]] || fail "a second run changed files" +ok "nothing to do" + +passed "migrate-main.sh: all checks passed."