From 4c20ade5389ecb89de21df0c85e7520ab31a0169 Mon Sep 17 00:00:00 2001 From: Austin Burdine Date: Fri, 9 Oct 2026 15:37:12 -0400 Subject: [PATCH 1/5] Added Tinybird sync credentials to the manager's layout ref https://linear.app/ghost/issue/PLA-482 main gained TINYBIRD_SYNC_AUTH (#332) after next-docker was cut from it, so the new compose.yml never passed it to Ghost or Traffic Analytics. Migrating a main installation would carry the setting in .env and silently stop using it. Ported here first so the migration from the released main layout preserves it. - compose.yml: passes TINYBIRD_SYNC_AUTH to Ghost as tinybird__sync_auth_key and to Traffic Analytics. - .env.example: documents it as an optional operator setting. - TINYBIRD.md: says how to generate and apply it. --- .env.example | 3 +++ TINYBIRD.md | 1 + compose.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.env.example b/.env.example index 252f5c68..3f9f402b 100644 --- a/.env.example +++ b/.env.example @@ -129,5 +129,8 @@ LOG_MAX_FILE="3" # TINYBIRD_TRACKER_TOKEN="p.eyJxxxxx" # TINYBIRD_ADMIN_TOKEN="p.eyJxxxxx" # TINYBIRD_WORKSPACE_ID="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" +# Shared secret Ghost uses to authenticate Tinybird sync requests to Traffic +# Analytics, for automation analytics. Generate one with `openssl rand -hex 32`. +# TINYBIRD_SYNC_AUTH="xxxxxxxx" # SALT_STORE_TYPE="file" # TRAFFIC_ANALYTICS_LOG_LEVEL="info" diff --git a/TINYBIRD.md b/TINYBIRD.md index 6f7f3ac1..fbd33618 100644 --- a/TINYBIRD.md +++ b/TINYBIRD.md @@ -8,6 +8,7 @@ Note: Currently Traffic Analytics features are behind a feature flag. For now, y 1. Run `docker compose run --rm tinybird-deploy` and wait for the service to exit successfully. This will create your Tinybird datasources, pipes and API endpoints. It may take a minute or two to complete the first time. You should see "Deployment #1 is live!" in your terminal before the service exits. 1. Run `docker compose run --rm tinybird-login get-tokens` 1. Copy and paste the values from the previous step into your `.env` file (Tinybird credentials are operator settings, not Ghost application settings) +1. If using automations analytics, generate a shared sync secret with `openssl rand -hex 32` and add it to `.env`: `./ghost-docker config set .env TINYBIRD_SYNC_AUTH `. Ghost and Traffic Analytics share it; after adding or changing it, run `docker compose up -d` to recreate their containers (a restart alone does not apply environment changes). 1. Run `docker compose --profile=analytics up -d` to start all services in the background 1. Add `analytics` to `COMPOSE_PROFILES` in your `.env` file, alongside the site mode, to include the `analytics` profile automatically when running `docker compose` commands. Profiles are additive: adding `analytics` does not change the site mode. 1. In production, add the analytics route to `caddy/sites/site.caddy`, inside the site's block: `import /etc/caddy/snippets/TrafficAnalytics traffic-analytics-:3000`, then reload Caddy with `docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile` diff --git a/compose.yml b/compose.yml index 24467c53..5bb90b0b 100644 --- a/compose.yml +++ b/compose.yml @@ -65,6 +65,7 @@ services: tinybird__tracker__datasource: analytics_events tinybird__adminToken: ${TINYBIRD_ADMIN_TOKEN:-} tinybird__workspaceId: ${TINYBIRD_WORKSPACE_ID:-} + tinybird__sync_auth_key: ${TINYBIRD_SYNC_AUTH:-} tinybird__stats__endpoint: ${TINYBIRD_API_URL:-https://api.tinybird.co} volumes: - ${UPLOAD_LOCATION:-./data/ghost}:${GHOST_CONTENT_PATH:-/home/ghost/content} @@ -194,6 +195,7 @@ services: SALT_STORE_TYPE: ${SALT_STORE_TYPE:-file} SALT_STORE_FILE_PATH: /data/salts.json TINYBIRD_TRACKER_TOKEN: ${TINYBIRD_TRACKER_TOKEN:-} + TINYBIRD_SYNC_AUTH: ${TINYBIRD_SYNC_AUTH:-} LOG_LEVEL: ${TRAFFIC_ANALYTICS_LOG_LEVEL:-info} networks: ghost_network: From 4b67c11d3963caa8216e9126c1a27c3843d3da99 Mon Sep 17 00:00:00 2001 From: Austin Burdine Date: Fri, 9 Oct 2026 15:37:21 -0400 Subject: [PATCH 2/5] =?UTF-8?q?=E2=9C=A8=20Moved=20installations=20of=20th?= =?UTF-8?q?e=20released=20main=20layout=20onto=20this=20one=20(S6b)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes https://linear.app/ghost/issue/PLA-482 Installations made from main update with git pull, and a pull across this layout breaks them: their .env selects no site mode, their Ghost image has a layout this release refuses, and their hand-edited Caddyfile reads environment variables the new Caddy no longer gets. This gates merging next-docker into main. The served launcher's self-update, run in such a checkout, finds no metadata and main's .env, and runs migration 0001-compose-profiles. It decides everything before changing anything (Compose resolves the staged configuration with the operator's overrides, Caddy loads the translated routes, git shows the stack files unedited), then follows self-update's recovery boundary with its snapshot, writer pause and a checked backup. Ghost keeps its exact version on that version's `next` image. The Caddyfile is translated rather than replaced so custom routes keep working. The metadata, written last, is the only record that the migration is done; no general migration framework was built. - manager/src/legacy.ts: the migration, its refusals and recovery. - manager/src/legacy/caddy.ts: the Caddyfile translation. - manager/src/legacy/config.ts: the .env / ghost.env split. - manager/src/commands/self-update.ts: enters the migration. - manager/src/backup.ts: a `migrating` backup that expects drift. - manager/src/recovery.ts: Snapshot, shared with self-update. - manager/src/compose.ts, commands/install.ts: variables of staged inputs; the loopback port search exported. - manager/test/: translation, split, and migration under fakes. - tests/e2e/migrate-main.sh, .github/workflows/test.yml: real migration, refusals and recovery on Linux CI. - tests/fixtures/released-main/: main's stack files. - docs/, README.md, .gitignore: operator steps, architecture, roadmap. --- .github/workflows/test.yml | 14 + .gitignore | 2 +- README.md | 5 +- docs/architecture.md | 17 +- docs/configuration.md | 5 +- docs/ghost-cli-replacement.md | 45 +- docs/install.md | 60 +- manager/src/backup.ts | 20 +- manager/src/commands/install.ts | 2 +- manager/src/commands/self-update.ts | 56 +- manager/src/compose.ts | 8 +- manager/src/legacy.ts | 901 ++++++++++++++++++ manager/src/legacy/caddy.ts | 269 ++++++ manager/src/legacy/config.ts | 111 +++ manager/src/recovery.ts | 43 +- manager/test/legacy.test.ts | 254 +++++ manager/test/migrate-main.test.ts | 383 ++++++++ manager/test/site.ts | 6 +- tests/e2e/migrate-main.sh | 288 ++++++ tests/fixtures/released-main/.env.example | 70 ++ tests/fixtures/released-main/README.md | 13 + .../released-main/caddy/Caddyfile.example | 60 ++ .../released-main/caddy/snippets/ActivityPub | 13 + .../released-main/caddy/snippets/Logging | 6 + .../caddy/snippets/SecurityHeaders | 12 + .../caddy/snippets/TrafficAnalytics | 6 + tests/fixtures/released-main/compose.ipv6.yml | 6 + tests/fixtures/released-main/compose.yml | 209 ++++ .../mysql-init/create-multiple-databases.sh | 18 + 29 files changed, 2809 insertions(+), 93 deletions(-) create mode 100644 manager/src/legacy.ts create mode 100644 manager/src/legacy/caddy.ts create mode 100644 manager/src/legacy/config.ts create mode 100644 manager/test/legacy.test.ts create mode 100644 manager/test/migrate-main.test.ts create mode 100755 tests/e2e/migrate-main.sh create mode 100644 tests/fixtures/released-main/.env.example create mode 100644 tests/fixtures/released-main/README.md create mode 100644 tests/fixtures/released-main/caddy/Caddyfile.example create mode 100644 tests/fixtures/released-main/caddy/snippets/ActivityPub create mode 100644 tests/fixtures/released-main/caddy/snippets/Logging create mode 100644 tests/fixtures/released-main/caddy/snippets/SecurityHeaders create mode 100644 tests/fixtures/released-main/caddy/snippets/TrafficAnalytics create mode 100644 tests/fixtures/released-main/compose.ipv6.yml create mode 100644 tests/fixtures/released-main/compose.yml create mode 100755 tests/fixtures/released-main/mysql-init/create-multiple-databases.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b8f3bc46..d53c8364 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -118,6 +118,19 @@ jobs: - name: Self-update end to end run: tests/e2e/self-update.sh + # An installation of the released main layout, migrated by the served + # launcher: refusals before any change, recovery after a failed start, and + # the migration keeping its project, data, routes and overrides. + migrate-main-linux: + name: Migration from the released main layout on Linux + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Migration end to end + run: tests/e2e/migrate-main.sh + # Real backups and restores on Docker Engine: a local site with ActivityPub # backed up, restored over itself and into a new directory, the lock # refusing a second operation, and a dump that fails. @@ -180,6 +193,7 @@ jobs: - launcher-linux - install-linux - self-update-linux + - migrate-main-linux - backup-linux - import-linux - launcher-macos diff --git a/.gitignore b/.gitignore index f455f042..a2c5e15c 100644 --- a/.gitignore +++ b/.gitignore @@ -98,7 +98,7 @@ caddy/custom/* caddy/global/* !caddy/global/.gitignore !caddy/global/README.md -# Operator's own Caddyfile from pre-1.0 installations (see the S6 migration) +# The released main layout's own Caddyfile, kept by its migration (docs/install.md) caddy/Caddyfile.local # The operator's own Compose overrides (docs/configuration.md) compose.override.yml diff --git a/README.md b/README.md index 642e448d..b967c9f6 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ Configuration to run Ghost and its services with Docker Compose. > a manager image: a small CLI in a container, started by a bash launcher that > uses Docker. It installs local and production sites, imports local Ghost-CLI > sites, backs up and restores them, and updates image installations between -> beta releases. Production import and migration from the released `main` layout -> remain on the [roadmap](docs/ghost-cli-replacement.md). For a supported setup +> beta releases, and moves installations of the released `main` layout onto it. +> Production import remains on the [roadmap](docs/ghost-cli-replacement.md). For a supported setup > today, use the `main` branch. ## The launcher @@ -91,6 +91,7 @@ From `scripts/`, run `pnpm install --frozen-lockfile`, `pnpm run typecheck` and tests/e2e/launcher.sh # stand-in Docker, then the real image tests/e2e/install.sh # real installations: pulls images, binds 80 and 443 tests/e2e/self-update.sh # release updates, failed-update write retention and recovery +tests/e2e/migrate-main.sh # an installation of the released main layout, migrated tests/e2e/backup.sh # real backups and restores, including ActivityPub tests/e2e/import.sh # real Ghost-CLI sites exported and imported ``` diff --git a/docs/architecture.md b/docs/architecture.md index 7d17ff17..f509f834 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -253,4 +253,19 @@ useful fresh/damaged-install diagnostics. Bundle v1 has its own S12 must record the exact schema/format/launcher versions supported by the stable release here. Subsequent changes must be compatible or explicitly versioned with a migration or old-format reader; pinned launchers must still start newer update -managers. Migration from the released `main` layout remains required in S6b. +managers. + +### Migration from the released main layout + +`src/legacy.ts` is migration `0001-compose-profiles`: the one way an installation +of the released `main` layout (a clone with no metadata) reaches this layout, +entered through the served launcher's `self-update`. `src/legacy/caddy.ts` +translates the operator's Caddyfile and `src/legacy/config.ts` splits its `.env`. +It decides everything before changing anything: Compose resolves the staged +configuration with the operator's overrides and Caddy loads the staged routes. +It then follows self-update's recovery boundary, reusing its snapshot, writer +pause and backup; the backup skips drift refusal because the stopped containers +deliberately ran main's images. The metadata is written last and is the only +record of completion: a site with it is never migrated again. There is no +general migration framework; a later migration from a released layout adds +what it needs. diff --git a/docs/configuration.md b/docs/configuration.md index a2acd538..bb2d901d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -295,8 +295,9 @@ minimum and exact source-version requirement. ## Existing installations Do not use `git pull` to move a released `main` installation to this layout. -That migration is still [roadmap work](ghost-cli-replacement.md#s6b--migration-from-the-released-main-layout). -Moving a Ghost-CLI site is a separate, supported local import described in +The served launcher's `self-update` migrates it, splitting its `.env` into +`.env` and `ghost.env` as this page describes; [install.md](install.md#moving-from-the-released-main-layout) +lists what each setting becomes. Moving a Ghost-CLI site is a separate, supported local import described in [install.md](install.md#importing-a-ghost-cli-site). ## Installed image pins diff --git a/docs/ghost-cli-replacement.md b/docs/ghost-cli-replacement.md index 4ca01763..a32a2ca8 100644 --- a/docs/ghost-cli-replacement.md +++ b/docs/ghost-cli-replacement.md @@ -11,8 +11,8 @@ Linear](https://linear.app/ghost/issue/PLA-412). | Outcome | Remaining work | Dependencies | | --- | --- | --- | -| Tagged single-site production | S6b legacy-layout migration, S5e production import, S12 qualification | Existing image self-update, local import, backup and restore | -| Several sites behind shared Caddy | S13 | Existing backup/restore; S6b for converting existing sites | +| Tagged single-site production | S5e production import, S12 qualification | Existing image self-update, local import, backup and restore | +| Several sites behind shared Caddy | S13 | Existing backup/restore and released-main migration | | Admin-driven Ghost upgrades | S7 host upgrade, S8 supervisor, S9 core adapter/API, S10 Admin | S7 before supervisor execution; S8 protocol before S9; S9 before S10; S13 integration if shipped | | Later extensions | S11 file secrets, S14 service references, S15 nightlies, S16 Redis | See each step | @@ -49,37 +49,6 @@ records, active theme, assets, redirects and configuration preserved. This repla `main`'s `scripts/migrate.sh`; S12 must not merge `next-docker` into `main` before these scenarios pass. -### S6b — Migration from the released main layout - -Repo: ghost-docker. Depends on image self-update and backup/restore. This migration -remains required regardless of the development-format [compatibility -policy](architecture.md#compatibility), and gates merging into `main`. - -Implement ordered release migration scripts, recording completion in metadata when the -first migration needs that field. `0001-compose-profiles` must: - -- Handle both absent profiles and existing `analytics,activitypub`, adding - `production`; preserve credentials and project identity. -- Split Ghost application configuration into `ghost.env`, keeping operator - settings in `.env`; add `SITE_MODE`, `URL`, `PROJECT_DIR` and an exact pin for - the currently running Ghost version. -- Preserve the existing untracked Caddyfile before managed files are written. - Custom routes must still work with snippets that take explicit upstream and - domain arguments. Preserve supported customizations automatically, or stop - before changing the live site and explain what must be resolved. - -Existing installations have no launcher or metadata. The entry point is the served -launcher from inside the installation (`curl -fsSL https://docker.ghost.org/install.sh | -bash -s -- self-update`), not a raw `git pull` across the breaking change. This is a -migration of the released layout, distinct from self-update of a current `source: -checkout` site. - -Acceptance: absent metadata, an untagged starting commit, existing optional profiles, -custom Caddy routes and Compose overrides, skipped releases, repeated invocation and -failed hooks. Failures during migration, pulling or startup must follow the [recovery -boundary](architecture.md#recovery) and report the observed state accurately, retaining -any writes after startup. - ### S7 — Host-driven Ghost upgrades Repo: ghost-docker. Depends on backup/restore and release compatibility. Implement @@ -250,13 +219,14 @@ environment-based installs, older imports, restart, and restore still work. ### S12 — Single-site release qualification and documentation -Repo: ghost-docker, with cross-repo fixtures. Deps: S5e and S6b, plus existing install, -backup/restore and self-update; qualify S13, S7-S10 and S11 when they have shipped. +Repo: ghost-docker, with cross-repo fixtures. Deps: S5e, plus existing install, +backup/restore, self-update and released-main migration; qualify S13, S7-S10 and S11 when they have shipped. Gates the first stable tag and merging `next-docker` into `main`. Include the launcher on Linux, macOS and WSL2. Consolidate CI and qualify the actual minimum supported tools and image versions. Run fresh local/production install, optional-service variants, CLI migration, legacy stack update, Ghost upgrade/recovery, supervisor/Admin, and restore -scenarios. Include Linux runtime tests and macOS-compatible shell/configuration checks. +scenarios. Refresh `tests/fixtures/released-main` from `main`'s final commit and rerun +`tests/e2e/migrate-main.sh` against it before merging. Include Linux runtime tests and macOS-compatible shell/configuration checks. Record the [compatibility baseline](architecture.md#compatibility): the exact metadata schema, backup format, launcher contract and bundle versions the stable release supports, and the obligations that hold for them from then on. @@ -271,7 +241,8 @@ shipped. ### S13 — Optional shared Caddy Repo: ghost-docker. Depends on backup/restore for new member sites; converting an -existing site into a member (PLA-504) also needs S6b. Several sites on one server, as +existing site into a member (PLA-504) starts from a site on this layout, which the +released-main migration provides. Several sites on one server, as Ghost-CLI ran several sites behind one nginx. 80 and 443 can belong to one Caddy only, so that Caddy is shared; everything else stays per site, MySQL included, so backup, restore, upgrade and import are unchanged. diff --git a/docs/install.md b/docs/install.md index fb45cef5..9f6882b0 100644 --- a/docs/install.md +++ b/docs/install.md @@ -452,7 +452,8 @@ site's launcher starts that image, not the one it is pinned to. | `--channel CHANNEL` | The newest release on `stable` or `beta`, which the site then follows. | | `--to vX.Y.Z` | Exactly that release. | -The update refuses downgrades, incompatible Ghost versions, absent metadata, +The update refuses downgrades, incompatible Ghost versions, absent metadata +(except for [the released main layout](#moving-from-the-released-main-layout)), checkouts, another operation's lock and an unfinished update snapshot. It pulls images before the outage where possible, keeps a file snapshot in `.ghost-docker-update/`, then pauses Ghost and ActivityPub and takes a consistent @@ -517,6 +518,63 @@ before, and restore the backup (`./ghost-docker restore --yes backups/`) the backup records the commit checked out when it was taken, and restore refuses a clone at any other. +### Moving from the released main layout + +An installation made from `main` before this layout (a git clone with one +`.env`, a hand-edited `caddy/Caddyfile`, and no `ghost-docker` launcher) moves +onto it once, with the launcher as docker.ghost.org serves it, run in the +site directory: + +```bash +cd /path/to/your/ghost-docker +curl -fsSL https://docker.ghost.org/install.sh | bash -s -- self-update --check +curl -fsSL https://docker.ghost.org/install.sh | bash -s -- self-update +``` + +**Do not `git pull` across this change.** The new layout's Compose file selects +no services without the new settings, and a pull cannot write them. + +`self-update` finds no `.ghost-docker.json` and main's `.env`, and runs migration +`0001-compose-profiles`. Ghost and its database must be running, so it can +read the Ghost version the site runs and back the site up. Before changing +anything it works out and checks all of this, and stops, naming what to +resolve, if any of it cannot be carried over: + +| What | Becomes | +| --- | --- | +| `COMPOSE_PROFILES` | `production`, plus `analytics` and `activitypub` if they were on | +| Compose project | The same name, set as `COMPOSE_PROJECT_NAME`, so Caddy's certificates and every volume stay the site's | +| `DOMAIN`, `ADMIN_DOMAIN` | `URL`, `ADMIN_URL` | +| Credentials, ports, data locations, Tinybird settings | Kept in `.env`, with `SITE_MODE`, `PROJECT_DIR` and the rest of this layout's settings added | +| Ghost's configuration in `.env` (`mail__*`, `labs__*`, anything Ghost read) | Moved to `ghost.env`, with the values Ghost actually received. Keys the container sets itself (`url`, `database__*`) are listed and left out, as they were overridden on main too | +| `ghost:6-alpine` | The `next` image of **exactly the running version**, pinned by digest (`GHOST_IMAGE_REF`). Ghost 6.61.0 is the first with one: upgrade an older Ghost on main first (`docker compose pull ghost && docker compose up -d`) | +| `caddy/Caddyfile` | Kept as `caddy/Caddyfile.local`, and carried into `caddy/sites/site.caddy`: `{$DOMAIN}`-style variables filled in, snippet imports given their absolute paths and arguments, bare upstreams (`ghost:2368`) pointed at the site's aliases. Everything else you wrote is kept. A global options block moves to `caddy/global/legacy.caddy`. Caddy loads the result before anything changes | +| `compose.override.yml`, `GD_COMPOSE_OVERRIDES` | Kept, and resolved with the new layout before anything changes | +| Stack files (`compose.yml`, snippets, `mysql-init/`) | The release's. Git must show them unedited: a change in the work tree or in a commit no remote has stops the migration. Move such changes into `compose.override.yml` or a `.caddy` file of your own afterwards | + +Then it keeps a copy of every file it writes in `.ghost-docker-update/`, stops +Ghost and ActivityPub, writes the new layout and `.ghost-docker.json`, takes a +checked backup into `backups/`, starts the site on the new layout and verifies +it through Caddy. Ghost is unavailable from the backup until the new layout +starts. + +Recovery follows [self-update](#self-update)'s: a failure before startup puts +main's files back and starts Ghost again in the same containers. A failure +after startup stops the services, puts main's files back, leaves the data as +it is, and says **the site needs you**. Ghost's version never changes, so +`docker compose up -d` starts main's layout on that data. The backup holds the +databases and content as they were before the migration started anything; +once the migration has run, `./ghost-docker restore --yes backups/` +puts them back. + +Afterwards the directory is a site installed from the image: update it with +`./ghost-docker self-update`, not git (its `.git` is left in place, and shows +the stack files as changed). Running the served command again is an ordinary +self-update. Compose files you used with `-f` by hand, such as +`compose.ipv6.yml`, are not seen by the manager: name them with +`GD_COMPOSE_OVERRIDES` when you migrate, as for every manager command +([configuration](configuration.md#the-compose-invocation-contract)). + ## backup and restore ```text diff --git a/manager/src/backup.ts b/manager/src/backup.ts index 5a7a0694..b29d389c 100644 --- a/manager/src/backup.ts +++ b/manager/src/backup.ts @@ -277,6 +277,14 @@ export interface BackupInput { * resumes them as soon as the capture is done. */ readonly pause?: WriterPause; + /** + * The configuration was just rewritten from the released main layout, with + * the writers stopped (legacy.ts): the stopped containers ran the old + * layout's images, which the configuration deliberately no longer names. + * Drift is not refused, and only services running exactly what the + * configuration names are recorded as having run it. + */ + readonly migrating?: boolean; } /** The manager taking the backup, as it was built. */ @@ -321,6 +329,7 @@ export async function takeBackup({ now = new Date(), consistent = false, pause, + migrating = false, }: BackupInput): Promise { const dir = site.dir; const databases = siteDatabases(site); @@ -329,7 +338,9 @@ export async function takeBackup({ const resolved = await io.busy('Resolving the Compose project', () => resolveSite(io, dir)); refuseMovedData(site, resolved); const overrides = siteOverrides(resolved); - await refuseDrift(io, resolved); + if (!migrating) { + await refuseDrift(io, resolved); + } const images: Record = {}; for (const [service, definition] of Object.entries(resolved.services)) { if (definition.image !== null) { @@ -381,7 +392,12 @@ export async function takeBackup({ // What runs, exactly: the backup records it beside what is configured, // and checks the dumps with the MySQL that wrote them. const running = await observeSite(io, resolved); - const ran = await runningImages(io, running); + const ran = await runningImages( + io, + migrating + ? running.filter((each) => resolved.services[each.service]?.image === each.image) + : running, + ); const commit = metadata.source === 'checkout' ? await checkedOut(io, dir) : null; io.stdout('\nThe capture\n'); diff --git a/manager/src/commands/install.ts b/manager/src/commands/install.ts index 67f6342c..fada7d3e 100644 --- a/manager/src/commands/install.ts +++ b/manager/src/commands/install.ts @@ -491,7 +491,7 @@ async function sitePorts( } /** The first port no container publishes and, where that can be told, nothing on the host holds. */ -async function freeOnHost( +export async function freeOnHost( io: Io, published: ReadonlySet, start = DEFAULT_PORT, diff --git a/manager/src/commands/self-update.ts b/manager/src/commands/self-update.ts index cda812a8..5e2a3020 100644 --- a/manager/src/commands/self-update.ts +++ b/manager/src/commands/self-update.ts @@ -9,10 +9,18 @@ import { z } from 'zod'; import { defineCommand, flag } from '../command.ts'; import { findingErrors, validate } from '../config.ts'; import { CliError, describeError, EXIT } from '../errors.ts'; -import { atomicWrite, copyPresent } from '../fs.ts'; +import { atomicWrite } from '../fs.ts'; import type { Io } from '../io.ts'; import { acquireLock } from '../lock.ts'; -import { isoSeconds, requireMetadata, siteFiles, writeMetadata, type Metadata } from '../meta.ts'; +import { isReleasedMainLayout, migrateReleasedMain } from '../legacy.ts'; +import { + isoSeconds, + readMetadata, + requireMetadata, + siteFiles, + writeMetadata, + type Metadata, +} from '../meta.ts'; import { LAUNCHER, launcherContent, @@ -21,7 +29,7 @@ import { sha256, stackDir, } from '../payload.ts'; -import { describeServices, runningServices, stopServices } from '../recovery.ts'; +import { describeServices, runningServices, Snapshot, stopServices } from '../recovery.ts'; import { compareReleases, isRelease } from '../release.ts'; import { resolveSite } from '../resolved.ts'; import { heading, ok, printChecks } from '../report.ts'; @@ -210,43 +218,6 @@ function writePayloadChanges(io: Io, dir: string, stack: string, payload: Payloa } } -// --- The snapshot --------------------------------------------------------------- - -/** Copies of what an update may change, to put back if it fails. */ -class Snapshot { - readonly dir: string; - readonly root: string; - readonly paths: string[]; - - constructor(dir: string, paths: string[]) { - this.dir = dir; - this.root = join(dir, UPDATE_DIR); - this.paths = paths; - } - - take(): void { - mkdirSync(join(this.root, 'files'), { recursive: true, mode: 0o700 }); - copyPresent(this.dir, this.paths, join(this.root, 'files')); - } - - /** Every path as it was: copied back, or removed when it did not exist. */ - restore(): void { - for (const path of this.paths) { - const target = join(this.dir, path); - const kept = join(this.root, 'files', path); - rmSync(target, { recursive: true, force: true }); - if (existsSync(kept)) { - mkdirSync(dirname(target), { recursive: true, mode: 0o755 }); - cpSync(kept, target, { recursive: true, preserveTimestamps: true }); - } - } - } - - remove(): void { - rmSync(this.root, { recursive: true, force: true }); - } -} - // --- Deciding ------------------------------------------------------------------- type Direction = 'newer' | 'current' | 'downgrade'; @@ -300,6 +271,11 @@ export const selfUpdateCommand = defineCommand({ const requested = requestedRelease(flags.channel, flags.to, '--to'); const { context, site } = installedSite(io); const dir = site.dir; + // An installation of the released main layout has no metadata: this is + // how it moves onto this one (legacy.ts). + if (readMetadata(dir).state === 'absent' && isReleasedMainLayout(site.settings)) { + return migrateReleasedMain(io, context, site, requested, flags.check); + } const metadata = requireMetadata(dir, 'update cannot tell which files are its own'); if (metadata.source === 'checkout') { throw new CliError( diff --git a/manager/src/compose.ts b/manager/src/compose.ts index 7c65c812..83dc8b34 100644 --- a/manager/src/compose.ts +++ b/manager/src/compose.ts @@ -365,8 +365,12 @@ const variables = z.record(z.string(), z.unknown()); * and every override in use, merged, so one an override removes the only use * of is not among them. `$$` is a literal and is not among them either. */ -export async function composeVariables(io: Io, dir: string): Promise | null> { - const result = await compose(io, { dir })`config --variables --format json`; +export async function composeVariables( + io: Io, + dir: string, + inputs?: ComposeInputs, +): Promise | null> { + const result = await compose(io, { dir, inputs })`config --variables --format json`; if (result.exitCode !== 0) { return null; } diff --git a/manager/src/legacy.ts b/manager/src/legacy.ts new file mode 100644 index 00000000..fb706cea --- /dev/null +++ b/manager/src/legacy.ts @@ -0,0 +1,901 @@ +// Migration 0001-compose-profiles: an installation of the released `main` +// layout, moved onto this one. It is how such a site comes to `self-update` +// (docs/install.md#moving-from-the-released-main-layout). +// +// A `main` installation is a git clone that updates with `git pull`. It has no +// launcher and no metadata; its compose.yml has no site-mode profiles; its one +// `.env` is both Compose's settings and Ghost's configuration; its Caddyfile +// is an untracked, hand-edited file; it runs a Ghost-CLI-layout Ghost image. +// The served launcher, run in it as `self-update`, starts this release's +// manager, which finds no metadata and that layout, and does the following: +// +// 1. Works out everything, changing nothing: the project's name, as Compose +// and the running containers have it, so volumes and certificates stay +// the site's; the Ghost version that runs, and the `next` image of +// exactly it; the new `.env`, `ghost.env` and routes, resolved by Compose +// with the operator's overrides and loaded by Caddy. Anything it cannot +// carry over stops it here, saying what to resolve. +// 2. Keeps a copy of every file it will write, stops Ghost (and +// ActivityPub), writes the release's files and the site's own, and takes +// a checked backup of the databases and content. +// 3. Starts the site on the new layout and verifies it. +// +// Recovery follows self-update's (docs/architecture.md#recovery): before +// startup is attempted the old files are put back and Ghost started again as +// it was; after, the services are stopped, the old files put back, and the +// data left as it is, for the operator. +// +// Completion is recorded by what it writes: the metadata. A site with it is +// never migrated again, so a second run is an ordinary self-update. +import { + copyFileSync, + cpSync, + existsSync, + mkdirSync, + readFileSync, + rmSync, + statSync, +} from 'node:fs'; +import { dirname, join, relative } from 'node:path'; +import { refuseMovedData, siteOverrides, takeBackup } from './backup.ts'; +import { renderRoutes, SITE_FILE } from './caddy.ts'; +import { freeOnHost } from './commands/install.ts'; +import { releaseOf, type ManagerRelease, type Requested } from './commands/common.ts'; +import { + compose, + composeConfig, + composeError, + composeVariables, + upAndWait, + type ComposeInputs, +} from './compose.ts'; +import { documentedVariables, findingErrors, validate } from './config.ts'; +import type { Context } from './context.ts'; +import { inspectImage, runOnce } from './docker/client.ts'; +import * as env from './env.ts'; +import { CliError, describeError, EXIT } from './errors.ts'; +import { atomicWrite, PRIVATE, readIfExists } from './fs.ts'; +import { MINIMUM_IMPORT_VERSION, resolveExactGhost, type ResolvedGhost } from './ghost.ts'; +import type { Io } from './io.ts'; +import { + GLOBAL_FILE, + KEPT_CADDYFILE, + LEGACY_CADDYFILE, + translateCaddyfile, + UntranslatableCaddyfile, +} from './legacy/caddy.ts'; +import { + readLegacyEnv, + refuseInterpolated, + splitLegacyEnv, + type SplitEnv, +} from './legacy/config.ts'; +import { acquireLock } from './lock.ts'; +import { isoSeconds, SCHEMA_VERSION, writeMetadata, type Metadata } from './meta.ts'; +import { + LAUNCHER, + launcherContent, + managerPin, + payloadFiles, + sha256, + stackDir, +} from './payload.ts'; +import { takenPorts } from './ports.ts'; +import { git } from './process.ts'; +import { describeServices, runningServices, Snapshot, stopServices } from './recovery.ts'; +import { observeSite, resolveConfig, resolveSite, type ResolvedSite } from './resolved.ts'; +import { heading, ok, printChecks } from './report.ts'; +import { + COMPOSE_FILE, + ENV_EXAMPLE_FILE, + ENV_FILE, + GHOST_ENV_FILE, + hostOf, + META_FILE, + OPERATOR_FILES, + readSettings, + siteFacts, + splitProfiles, + UPDATE_DIR, + type SiteFacts, + type SiteSettings, +} from './site.ts'; +import { verifySite } from './verify.ts'; +import { atLeast } from './versions.ts'; +import { WriterPause } from './writers.ts'; + +export const MIGRATION = '0001-compose-profiles'; +const FROM = 'the released main layout'; + +/** How long pulling the release's images may take. */ +const PULL_MS = 30 * 60 * 1000; +/** The optional profiles `main` had. */ +const MAIN_PROFILES = ['analytics', 'activitypub']; +const HOSTED_ACTIVITYPUB = 'https://ap.ghost.org'; +/** Where preflight stages the new `.env` and Caddy files, inside the site so the daemon sees them. */ +const STAGED = 'staged'; + +/** + * A site with no metadata whose `.env` is `main`'s: a DOMAIN, and none of the + * settings every site of this layout has. + */ +export const isReleasedMainLayout = (settings: SiteSettings): boolean => + settings.get('DOMAIN') !== undefined && + settings.get('URL') === undefined && + settings.get('SITE_MODE') === undefined; + +function refuse(message: string): never { + throw new CliError(`${message}\n Nothing has been changed.`); +} + +/** Everything decided before the first change. */ +interface Plan { + readonly io: Io; + readonly dir: string; + readonly release: ManagerRelease; + /** The manager image the site's launcher is pinned to. */ + readonly pin: string; + readonly project: string; + readonly profiles: string[]; + readonly domain: string; + readonly adminDomain: string; + readonly ghost: ResolvedGhost; + /** The services running before the migration, which it stops and may start again. */ + readonly running: string[]; + readonly stack: string; + /** The release's files, relative to the stack. */ + readonly files: string[]; + readonly env: string; + readonly ghostEnv: string; + readonly split: SplitEnv; + readonly routes: string; + readonly global: string | null; + readonly caddyChanges: readonly string[]; + readonly hadCaddyfile: boolean; + /** Overrides other than compose.override.yml, relative to the site. */ + readonly overrides: string[]; + readonly port: number; +} + +/** + * `self-update` in a site of the released main layout. Returns the exit + * status, or throws a CliError. + */ +export async function migrateReleasedMain( + io: Io, + context: Context, + site: SiteFacts, + requested: Requested, + check: boolean, +): Promise { + const dir = site.dir; + io.stdout( + `${dir} is an installation of ${FROM}, with no ${META_FILE}.\n` + + `Moving it onto this release's layout (migration ${MIGRATION}).\n`, + ); + if (context.source !== 'image') { + refuse( + 'this manager was built from a checkout, and the migration writes the release’s files\n' + + ' from a published manager image. Run the served launcher in the site directory:\n' + + ' curl -fsSL https://docker.ghost.org/install.sh | bash -s -- self-update', + ); + } + if (existsSync(join(dir, UPDATE_DIR))) { + refuse( + `${join(dir, UPDATE_DIR)} is left from a migration or update that did not finish, and holds\n` + + ' the files it would have put back. Once the site is as it should be, remove it and run this again.', + ); + } + + const staged = join(dir, UPDATE_DIR, STAGED); + let plan: Plan; + try { + plan = await prepare(io, context, site, requested, staged); + } finally { + rmSync(join(dir, UPDATE_DIR), { recursive: true, force: true }); + } + if (check) { + return report(plan); + } + const lock = acquireLock(dir, `migration ${MIGRATION} from ${FROM}`); + try { + return await apply(plan); + } finally { + lock.release(); + } +} + +// --- Working it out -------------------------------------------------------------- + +async function prepare( + io: Io, + context: Context, + site: SiteFacts, + requested: Requested, + staged: string, +): Promise { + const dir = site.dir; + const release = releaseOf(requested, io.env); + const text = readFileSync(join(dir, ENV_FILE), 'utf8'); + const legacy = readLegacyEnv(text); + const old = legacy.values; + if (old.COMPOSE_FILE !== undefined) { + refuse( + '.env sets COMPOSE_FILE, which the manager does not use: it names Compose files itself.\n' + + ' Remove it, and name any extra file with GD_COMPOSE_OVERRIDES when you run this again\n' + + ' (docs/configuration.md#the-compose-invocation-contract).', + ); + } + const domain = (old.DOMAIN ?? '').toLowerCase(); + const adminDomain = (old.ADMIN_DOMAIN ?? '').toLowerCase(); + for (const [key, value] of [ + ['DOMAIN', domain], + ['ADMIN_DOMAIN', adminDomain], + ] as const) { + if ((key === 'DOMAIN' || value !== '') && hostOf(`https://${value}/`) !== value) { + refuse(`.env's ${key} is ${JSON.stringify(value)}, which is not a domain.`); + } + } + const oldProfiles = splitProfiles(old.COMPOSE_PROFILES ?? ''); + const unknown = oldProfiles.filter((profile) => !MAIN_PROFILES.includes(profile)); + if (unknown.length > 0) { + refuse( + `COMPOSE_PROFILES in .env names ${unknown.join(', ')}; ${FROM} had only ${MAIN_PROFILES.join(' and ')}.`, + ); + } + for (const key of ['DATABASE_PASSWORD', 'DATABASE_ROOT_PASSWORD']) { + if (!old[key]) { + refuse(`.env has no ${key}, which ${FROM} required.`); + } + } + + heading(io, `Checking the installation`); + const stack = stackDir(io.env); + const files = payloadFiles(stack); + await refuseEditedStack(io, dir, files); + ok(io, 'stack files', 'unedited, so the release’s replace them'); + + // The old layout, as Compose resolves it with the operator's overrides: + // the project's name, and what Ghost received. + const before = await io.busy('Resolving the Compose project', () => resolveSite(io, dir)); + const project = before.project; + const oldConfig = await composeConfig(io, dir); + const oldVariables = await composeVariables(io, dir); + if (project === '' || !oldConfig.ok || oldVariables === null) { + refuse( + 'Compose cannot resolve the installation as it is: check it with docker compose config.', + ); + } + const received = Object.fromEntries( + Object.entries(oldConfig.project.services.ghost?.environment ?? {}).map(([key, value]) => [ + key, + value ?? '', + ]), + ); + const overrides = siteOverrides(before); + ok(io, 'project', `${project}, kept, with its volumes and Caddy’s certificates`); + + const containers = await observeSite(io, before); + const running = containers + .filter((each) => each.state === 'running') + .map((each) => each.service); + const ghostContainer = containers.find( + (each) => each.service === 'ghost' && each.state === 'running', + ); + if (ghostContainer === undefined || !running.includes('db')) { + refuse( + 'Ghost and its database must be running, so the version Ghost runs can be kept and its data\n' + + ' backed up. Start the site as it is (docker compose up -d), then run this again.', + ); + } + const image = await inspectImage(io.docker, ghostContainer.imageId); + const version = image?.env.GHOST_VERSION; + if (!image?.repoDigests.some((digest) => digest.startsWith('ghost@')) || !version) { + refuse( + `Ghost runs ${ghostContainer.image}, which is not the official ghost image, so there is no image of\n` + + ' this layout to move it to. Run the official image (remove the override that changes it).', + ); + } + if (!atLeast(version, MINIMUM_IMPORT_VERSION)) { + refuse( + `this site runs Ghost ${version}, and this layout's images start at Ghost ${MINIMUM_IMPORT_VERSION}.\n` + + ' A migration never changes Ghost. Upgrade it on the layout it runs now, then run this again:\n' + + ' docker compose pull ghost && docker compose up -d', + ); + } + ok(io, 'ghost', `${version}, running`); + + heading(io, 'Resolving the images'); + const ghost = await io.busy(`Resolving the image of Ghost ${version}`, () => + resolveExactGhost(io, version), + ); + ok( + io, + 'ghost', + `${ghost.image}:${ghost.tag}, ${ghost.reference}: the same version, this layout's image`, + ); + const pin = await io.busy('Resolving the manager image', () => managerPin(io, context)); + + // Ghost was not published on main; here it is, on the loopback interface. + const port = await freeOnHost( + io, + (await takenPorts(io)).published, + Number(old.GHOST_PORT) > 0 ? Number(old.GHOST_PORT) : undefined, + ); + const profiles = ['production', ...oldProfiles]; + const generated: [string, string][] = [ + ['COMPOSE_PROFILES', profiles.join(',')], + ['SITE_MODE', 'production'], + ['COMPOSE_PROJECT_NAME', project], + ['PROJECT_DIR', dir], + ['NODE_ENV', 'production'], + ['URL', `https://${domain}`], + ...(adminDomain ? [['ADMIN_URL', `https://${adminDomain}`] as [string, string]] : []), + ['GHOST_IMAGE', ghost.image], + ['GHOST_VERSION', ghost.tag], + ['GHOST_IMAGE_REF', ghost.reference], + ['GHOST_CONTENT_PATH', ghost.contentPath], + ['GHOST_TINYBIRD_PATH', ghost.tinybirdPath], + ['GHOST_PORT', String(port)], + ['RESTART_POLICY', 'unless-stopped'], + ['HTTP_PORT', old.HTTP_PORT || '80'], + ['HTTPS_PORT', old.HTTPS_PORT || '443'], + ['DATABASE_HOST', 'db'], + ['DATABASE_PORT', '3306'], + ['DATABASE_NAME', 'ghost'], + ['DATABASE_USER', old.DATABASE_USER || 'ghost'], + ['DATABASE_PASSWORD', old.DATABASE_PASSWORD!], + ['DATABASE_ROOT_PASSWORD', old.DATABASE_ROOT_PASSWORD!], + ['DATABASE_EXTRA_DATABASES', 'activitypub'], + ['UPLOAD_LOCATION', old.UPLOAD_LOCATION || './data/ghost'], + ['MYSQL_DATA_LOCATION', old.MYSQL_DATA_LOCATION || './data/mysql'], + ]; + + // The new layout, as Compose would resolve it with these settings and the + // operator's overrides, staged where Compose and the daemon can read it. + mkdirSync(staged, { recursive: true, mode: 0o700 }); + const inputs: ComposeInputs = { + files: [join(stack, COMPOSE_FILE), ...before.files.slice(1)], + envFile: join(staged, ENV_FILE), + }; + atomicWrite(inputs.envFile, env.serializeAll(generated), PRIVATE); + const draft = await composeConfig(io, dir, inputs); + const newVariables = await composeVariables(io, dir, inputs); + if (!draft.ok || newVariables === null) { + refuse( + `Compose cannot resolve this release's layout with the site's settings and overrides:\n ${draft.ok ? 'its variables could not be listed' : draft.reason}`, + ); + } + const documented = new Set([ + ...documentedVariables(readIfExists(join(dir, ENV_EXAMPLE_FILE)) ?? ''), + ...documentedVariables(readFileSync(join(stack, ENV_EXAMPLE_FILE), 'utf8')), + ]); + const isOperatorKey = (key: string) => + key.startsWith('COMPOSE_') || + oldVariables.has(key) || + newVariables.has(key) || + documented.has(key); + refuseInterpolated(text, new Set(legacy.keys.filter(isOperatorKey))); + const split = splitLegacyEnv({ + legacy, + generated: new Set(generated.map(([key]) => key)), + isOperatorKey, + isInterpolated: (key) => newVariables.has(key), + container: new Set(Object.keys(draft.project.services.ghost?.environment ?? {})), + received, + }); + const envText = env.serializeAll( + [...generated, ...split.operator], + `# Site settings, moved from ${FROM} by migration ${MIGRATION}. See .env.example\n` + + '# for the optional ones. Write values with ./ghost-docker config set .env KEY VALUE,\n' + + '# which encodes them for Compose. Ghost’s own configuration is in ghost.env.\n', + ); + atomicWrite(inputs.envFile, envText, PRIVATE); + const ghostEnv = env.serializeAll( + split.ghost, + `# Ghost application settings for ${project}, moved from .env by migration\n` + + `# ${MIGRATION}; see ghost.env.example. Write values with ./ghost-docker config set\n` + + '# ghost.env KEY VALUE, which encodes them for Compose.\n\n', + ); + + const after = await resolveConfig(io, dir, inputs); + if (after.services.ghost?.image !== ghost.reference) { + refuse( + `with the site's overrides, the ghost service would run ${after.services.ghost?.image ?? 'nothing'}, not\n` + + ` ${ghost.reference}. Remove the override that sets its image, then run this again.`, + ); + } + const draftSite = siteFacts(dir, { + get: (key) => env.toRecord(envText)[key], + }); + try { + refuseMovedData(draftSite, after); + } catch (error) { + refuse( + `${(error as Error).message.replace(/ Nothing has been changed\.$/, '')}\n` + + ' The migration backs the site up first, and a backup handles data only in ./data.', + ); + } + ok(io, 'configuration', `.env and ghost.env for a production site, resolved by Compose`); + + // The routes, loaded by the Caddy image the site will run. + // Where main sent ActivityPub, which with the profile on was still the + // hosted service unless ACTIVITYPUB_TARGET said otherwise: moving it would + // move the site's followers. + const activitypub = (old.ACTIVITYPUB_TARGET || HOSTED_ACTIVITYPUB).replace( + /^(?:https?:\/\/)?activitypub:8080$/, + `activitypub-${project}:8080`, + ); + const original = readIfExists(join(dir, LEGACY_CADDYFILE)); + let routes: string; + let global: string | null = null; + let caddyChanges: readonly string[] = []; + if (original === undefined) { + routes = renderRoutes({ + project, + domain, + adminDomain, + email: '', + activitypub: activitypub === `activitypub-${project}:8080`, + }); + } else { + try { + ({ + site: routes, + global, + changes: caddyChanges, + } = translateCaddyfile(original, { + project, + domain, + adminDomain, + activitypub, + })); + } catch (error) { + if (error instanceof UntranslatableCaddyfile) { + refuse( + `${error.message}\n Change caddy/Caddyfile so it can be carried over, or move what it does into\n` + + ' compose.override.yml or caddy/custom/ afterwards (docs/caddy.md).', + ); + } + throw error; + } + } + await validateRoutes(io, staged, stack, after, routes, global); + ok( + io, + SITE_FILE, + original === undefined + ? 'written: there was no caddy/Caddyfile' + : `carried over from caddy/Caddyfile, and loaded by Caddy`, + ); + + return { + io, + dir, + release, + pin, + project, + profiles, + domain, + adminDomain, + ghost, + running: [...new Set(running)], + stack, + files, + env: envText, + ghostEnv, + split, + routes, + global, + caddyChanges, + hadCaddyfile: original !== undefined, + overrides, + port, + }; +} + +/** + * The release replaces every stack file `main` tracked, so one the operator + * changed would be lost. Git, which installed the site, says which they are: + * changed in the work tree, or by a commit no remote has. + */ +async function refuseEditedStack(io: Io, dir: string, files: readonly string[]): Promise { + const top = await git(io, dir, ['rev-parse', '--show-toplevel']); + if (!top.ok || top.stdout.trim() !== dir) { + refuse( + `${dir} is not a git checkout. ${FROM} is installed with git clone, and without its history it\n` + + ' cannot be told whether compose.yml or the Caddy snippets were edited, which the release replaces.', + ); + } + const tracked = await git(io, dir, ['ls-files', '--', ...files]); + const paths = tracked.stdout.split('\n').filter(Boolean); + if (!tracked.ok || paths.length === 0) { + refuse( + 'git does not list the stack’s files in this checkout, so it is not one of ' + + FROM + + '.', + ); + } + const status = await git(io, dir, [ + 'status', + '--porcelain', + '--untracked-files=no', + '--', + ...paths, + ]); + const remotes = await git(io, dir, ['for-each-ref', '--format=%(refname)', 'refs/remotes']); + if (!status.ok || !remotes.ok) { + refuse(`git cannot read the checkout: ${status.stderr || remotes.stderr}`); + } + if (remotes.stdout.trim() === '') { + refuse( + 'the checkout has no remote-tracking branches, so its own commits cannot be told apart from\n' + + ' released ones. Fetch from https://github.com/TryGhost/ghost-docker.git, then run this again.', + ); + } + const local = await git(io, dir, [ + 'log', + '--format=', + '--name-only', + 'HEAD', + '--not', + '--remotes', + '--', + ...paths, + ]); + const edited = [ + ...new Set([ + ...status.stdout + .split('\n') + .filter((line) => line.trim() !== '') + .map((line) => line.slice(3)), + ...local.stdout.split('\n').filter(Boolean), + ]), + ].sort(); + if (edited.length > 0) { + refuse( + `these files of the stack were changed here, and the release replaces them:\n` + + edited.map((file) => ` ${file}\n`).join('') + + ' Move each change into compose.override.yml (Compose) or a .caddy file of your own\n' + + ' (docs/caddy.md) once the site is migrated, put the files back (git checkout -- ),\n' + + ' and run this again.', + ); + } +} + +/** + * The routes as the stack's Caddyfile imports them, loaded by `caddy + * validate` in the image the site will run, with no network. What it refuses, + * a reload would too. + */ +async function validateRoutes( + io: Io, + staged: string, + stack: string, + after: ResolvedSite, + routes: string, + global: string | null, +): Promise { + const caddy = join(staged, 'caddy'); + cpSync(join(stack, 'caddy'), caddy, { recursive: true }); + atomicWrite(join(staged, SITE_FILE), routes, 0o644); + if (global !== null) { + atomicWrite(join(staged, GLOBAL_FILE), global, 0o644); + } + const image = after.services.caddy?.image; + if (!image) { + refuse('Compose resolves no caddy service for the production layout.'); + } + const pulled = await io.busy( + 'Pulling the release’s images, while the site keeps running', + () => + compose(io, { + dir: after.dir, + inputs: { files: after.files, envFile: join(staged, ENV_FILE) }, + timeout: PULL_MS, + })`pull --quiet --ignore-buildable`, + ); + if (pulled.exitCode !== 0) { + refuse(`the release's images could not be pulled: ${composeError(pulled)}`); + } + const validated = await io.busy('Loading the routes in Caddy', () => + runOnce(io.docker, { + image, + entrypoint: ['caddy'], + cmd: ['validate', '--config', '/etc/caddy/Caddyfile', '--adapter', 'caddyfile'], + binds: [{ source: caddy, target: '/etc/caddy', readOnly: true }], + network: 'none', + }), + ); + if (validated.status !== 0) { + const said = (validated.stderr || validated.stdout).trim().split('\n').slice(-6); + refuse( + 'Caddy does not load the routes carried over from caddy/Caddyfile:\n' + + said.map((line) => ` ${line}\n`).join('') + + ' Change caddy/Caddyfile so they load, then run this again.', + ); + } +} + +// --- --check ----------------------------------------------------------------------- + +function report(plan: Plan): number { + const { io, split } = plan; + io.stdout( + [ + '', + `The migration would move this site onto ${describeRelease(plan.release)}, and change nothing else:`, + ` Ghost ${plan.ghost.version}, from the image of this layout: ${plan.ghost.reference}`, + ` Project ${plan.project} (${plan.profiles.join(',')}), its volumes kept`, + ` .env operator settings; ${split.operator.length} carried besides the generated ones`, + ` ghost.env ${split.ghost.length} Ghost settings, from .env`, + ...split.dropped.map( + ({ key, reason }) => ` ${key} not carried: ${reason}`, + ), + ` Routes ${SITE_FILE}${plan.hadCaddyfile ? ', from caddy/Caddyfile, kept as caddy/Caddyfile.local' : ''}`, + ...plan.caddyChanges.map((change) => ` ${change}`), + ` Loopback Ghost published on 127.0.0.1:${plan.port}`, + ` Launcher ./${LAUNCHER}, pinned to ${plan.pin}`, + '', + 'Ghost would be stopped for a backup, then the site started on the new layout.', + '', + ].join('\n'), + ); + return EXIT.ok; +} + +const describeRelease = (release: ManagerRelease): string => + release.version ?? 'this manager’s release'; + +// --- Changing it ------------------------------------------------------------------- + +/** Every path the migration writes, relative to the site: what its snapshot holds. */ +function touched(plan: Plan): string[] { + return [ + ...new Set([...OPERATOR_FILES, ...plan.overrides, ...plan.files, LAUNCHER, KEPT_CADDYFILE]), + ]; +} + +async function apply(plan: Plan): Promise { + const { io, dir } = plan; + heading(io, 'Keeping the current files'); + const snapshot = new Snapshot(dir, touched(plan)); + snapshot.take(); + ok(io, UPDATE_DIR, `.env, the Caddyfile, and the files the migration writes`); + + const pause = new WriterPause(io, dir, 'the migration'); + let servicesChanged = false; + let backup: string | null = null; + try { + heading(io, 'Stopping Ghost'); + await pause.stop(plan.running); + + heading(io, 'Writing the new layout'); + const metadata = write(plan); + + const findings = await io.busy('Validating the configuration', () => validate(io, dir)); + const errors = findingErrors(findings); + if (errors) { + throw new CliError(`the migrated configuration does not validate:\n${errors}`); + } + ok(io, 'configuration', 'valid'); + + heading(io, 'Backing up the site'); + backup = await takeBackup({ + io, + site: siteFacts(dir, readSettings(dir)!), + metadata, + consistent: true, + pause, + migrating: true, + }); + ok(io, 'backup', `${relative(dir, backup)}, checked`); + + heading(io, 'Starting the site'); + const pull = await io.busy( + 'Pulling the images this release names', + () => compose(io, { dir, timeout: PULL_MS })`pull --quiet --ignore-buildable`, + ); + if (pull.exitCode !== 0) { + throw new CliError(`the images could not be pulled: ${composeError(pull)}`); + } + // Set before attempting up: even a failed start may migrate data or accept writes. + servicesChanged = true; + pause.end(); + await upAndWait(io, dir, 'Starting the services and waiting for them to be healthy'); + ok(io, 'services', 'healthy, by their own health checks'); + + await verifySite(io, dir); + } catch (error) { + return recover(plan, snapshot, pause, backup, servicesChanged, error); + } + snapshot.remove(); + summarize(plan, backup!); + return EXIT.ok; +} + +/** The release's files, the site's own, and the metadata, written last. */ +function write(plan: Plan): Metadata { + const { io, dir, stack, files } = plan; + const checksums: Record = {}; + + if (plan.hadCaddyfile) { + copyFileSync(join(dir, LEGACY_CADDYFILE), join(dir, KEPT_CADDYFILE)); + ok(io, KEPT_CADDYFILE, 'your caddy/Caddyfile, kept for reference'); + } + for (const file of files) { + const source = join(stack, file); + const target = join(dir, file); + const content = readFileSync(source); + mkdirSync(dirname(target), { recursive: true, mode: 0o755 }); + atomicWrite(target, content, statSync(source).mode & 0o777); + checksums[file] = sha256(content); + } + ok(io, 'stack files', `${files.length} files of ${describeRelease(plan.release)}`); + const launcher = launcherContent(io.env, { image: plan.pin, channel: plan.release.channel }); + atomicWrite(join(dir, LAUNCHER), launcher, 0o755); + checksums[LAUNCHER] = sha256(launcher); + ok(io, LAUNCHER, `the launcher, pinned to ${plan.pin}`); + + atomicWrite(join(dir, ENV_FILE), plan.env, PRIVATE); + ok(io, ENV_FILE, 'operator settings, with the same credentials and project'); + atomicWrite(join(dir, GHOST_ENV_FILE), plan.ghostEnv, PRIVATE); + ok(io, GHOST_ENV_FILE, `${plan.split.ghost.length} Ghost settings, moved from .env`); + for (const { key, reason } of plan.split.dropped) { + printChecks(io, [{ status: 'note', label: 'not carried', detail: `${key}: ${reason}` }]); + } + atomicWrite(join(dir, SITE_FILE), plan.routes, 0o644); + if (plan.global !== null) { + atomicWrite(join(dir, GLOBAL_FILE), plan.global, 0o644); + } + ok(io, SITE_FILE, 'the site’s routes; yours to edit'); + + const metadata: Metadata = { + schemaVersion: SCHEMA_VERSION, + installedAt: isoSeconds(), + updatedAt: null, + mode: 'production', + channel: plan.release.channel, + source: 'image', + stack: { + version: plan.release.version, + ref: plan.release.version, + image: plan.pin, + previous: null, + }, + site: { + project: plan.project, + dir, + url: `https://${plan.domain}`, + domain: plan.domain, + adminDomain: plan.adminDomain || null, + }, + ghost: { + image: plan.ghost.image, + tag: plan.ghost.tag, + version: plan.ghost.version, + digest: plan.ghost.digest, + }, + profiles: plan.profiles, + payload: checksums, + }; + writeMetadata(dir, metadata); + ok(io, META_FILE, `installation metadata: migration ${MIGRATION} is done once this is written`); + return metadata; +} + +/** Before startup: the old files back, Ghost started again. After: the operator decides. */ +async function recover( + plan: Plan, + snapshot: Snapshot, + pause: WriterPause, + backup: string | null, + servicesChanged: boolean, + error: unknown, +): Promise { + const { io, dir } = plan; + io.stderr(`\n${describeError(error)}\n`); + io.stderr(`\nThe migration did not complete. Putting ${FROM}'s files back\n`); + const problems: string[] = []; + if (servicesChanged) { + const stopped = await stopServices(io, dir, 'Stopping the migrated site'); + if (stopped.error !== null) { + problems.push(`the services could not be stopped: ${stopped.error}`); + } + } + if (problems.length === 0) { + try { + snapshot.restore(); + } catch (restoreError) { + problems.push(`the files could not be put back: ${(restoreError as Error).message}`); + } + } + const resumed = [...pause.paused]; + if (problems.length === 0 && !servicesChanged) { + try { + await pause.resume(); + } catch (resumeError) { + problems.push((resumeError as Error).message); + } + } + + const kept = + backup === null + ? [] + : [`The backup taken for the migration is kept in ${relative(dir, backup)}.`]; + if (servicesChanged || problems.length > 0) { + const running = await runningServices(io, dir); + const filesBack = problems.length === 0; + io.stderr( + [ + '', + 'The site needs you.', + ...problems.map((problem) => ` ${problem}`), + ...(servicesChanged + ? [ + 'The new layout’s services started before the migration failed, and Ghost may have accepted', + `writes since the backup. Nothing was loaded over them. Ghost ${plan.ghost.version} ran on the data`, + 'either way: the migration never changes Ghost’s version.', + ] + : []), + describeServices(running), + ...(filesBack ? [`The files are ${FROM}'s again.`] : []), + '', + ...(filesBack + ? [ + 'Start the site as it was, on its data as it is: docker compose up -d', + ...(backup === null + ? [] + : [ + `The databases and content as they were before the migration started anything are in`, + `${relative(dir, backup)} (checked). Once the cause is fixed and the migration has run,`, + `./ghost-docker restore --yes ${relative(dir, backup)} puts them back.`, + ]), + ] + : kept), + '', + `The files as they were before the migration are in ${snapshot.root}/files.`, + `Once the site is as it should be, remove ${snapshot.root}.`, + '', + ].join('\n'), + ); + throw new CliError('the migration failed, and the site needs the operator.'); + } + snapshot.remove(); + io.stderr( + [ + '', + `Restored: the site is on ${FROM} again, with its files as they were` + + (resumed.length > 0 + ? `. ${resumed.join(' and ')}, stopped for the migration, ${resumed.length > 1 ? 'are' : 'is'} running again.` + : '.'), + ...kept, + '', + ].join('\n'), + ); + throw new CliError(`the migration failed; ${FROM} was restored.`); +} + +function summarize(plan: Plan, backup: string): void { + const { io, dir } = plan; + io.stdout( + [ + '', + `Moved from ${FROM} to ${describeRelease(plan.release)}.`, + '', + ` Site https://${plan.domain}`, + ` Ghost ${plan.ghost.version}, ${plan.ghost.reference}, the same version`, + ` Project ${plan.project} (${plan.profiles.join(',')})`, + ` Loopback 127.0.0.1:${plan.port}`, + ` Backup ${relative(dir, backup)}, of the site before it started on the new layout`, + '', + 'From now on:', + ' - Update the stack with ./ghost-docker self-update, never git pull: this directory', + ' is now installed from the manager image, and its stack files are the release’s.', + ' - Ghost’s own settings are in ghost.env; .env holds the rest.', + ` - The routes are in ${SITE_FILE}${plan.hadCaddyfile ? `; your old Caddyfile is ${KEPT_CADDYFILE}` : ''}.`, + ...(plan.global === null ? [] : [` - Its global options are in ${GLOBAL_FILE}.`]), + '', + ].join('\n'), + ); +} diff --git a/manager/src/legacy/caddy.ts b/manager/src/legacy/caddy.ts new file mode 100644 index 00000000..85ae4130 --- /dev/null +++ b/manager/src/legacy/caddy.ts @@ -0,0 +1,269 @@ +// The operator's own Caddyfile from the released `main` layout, carried into +// this layout's routes (docs/install.md#moving-from-the-released-main-layout). +// +// On `main`, caddy/Caddyfile was copied from Caddyfile.example and then edited +// by hand. It read DOMAIN, ADMIN_DOMAIN and ACTIVITYPUB_TARGET from Caddy's +// environment, imported snippets by relative path with no arguments, and +// proxied to bare service names. This layout gives Caddy no environment, its +// snippets take arguments, and upstreams are the site's unique aliases. The +// translation changes exactly those things and keeps everything else the +// operator wrote, so their routes keep working; Caddy itself then validates +// the result before anything live is changed. +import { join } from 'node:path'; + +/** The original, kept beside the routes it became; .gitignore leaves it out. */ +export const LEGACY_CADDYFILE = join('caddy', 'Caddyfile'); +export const KEPT_CADDYFILE = join('caddy', 'Caddyfile.local'); +/** A global options block, which this layout's Caddyfile imports from caddy/global/. */ +export const GLOBAL_FILE = join('caddy', 'global', 'legacy.caddy'); + +/** Services whose bare `name:port` is rewritten to the site's unique alias. */ +const ALIASED = ['ghost', 'db', 'traffic-analytics', 'activitypub'] as const; + +/** What the old Caddy container's environment held, as Compose set it on `main`. */ +export interface LegacyCaddyValues { + readonly project: string; + readonly domain: string; + readonly adminDomain: string; + /** ACTIVITYPUB_TARGET, already pointing at the site's alias when it was `activitypub:8080`. */ + readonly activitypub: string; +} + +export interface TranslatedCaddyfile { + /** caddy/sites/site.caddy. */ + readonly site: string; + /** The global options block's body, for GLOBAL_FILE; null when there was none. */ + readonly global: string | null; + /** What was changed, for the operator. */ + readonly changes: readonly string[]; +} + +/** A Caddyfile this cannot carry over automatically; nothing has been changed. */ +export class UntranslatableCaddyfile extends Error {} + +/** Snippets the stack ships, and the arguments each now takes, in order. */ +function snippetArguments(name: string, values: LegacyCaddyValues): string | null { + switch (name) { + case 'Logging': + return ''; + case 'TrafficAnalytics': + return ` traffic-analytics-${values.project}:3000`; + case 'ActivityPub': + return ` ${values.activitypub}`; + case 'SecurityHeaders': + return ` "${values.adminDomain}"`; + default: + return null; + } +} + +/** Splits a line into its code and its comment: `#` starts one only at a token's start. */ +function splitComment(line: string): { code: string; comment: string } { + let quoted = false; + for (let i = 0; i < line.length; i += 1) { + const char = line[i]!; + if (char === '\\') { + i += 1; + } else if (char === '"') { + quoted = !quoted; + } else if (char === '#' && !quoted && (i === 0 || /\s/.test(line[i - 1]!))) { + return { code: line.slice(0, i), comment: line.slice(i) }; + } + } + return { code: line, comment: '' }; +} + +/** + * `{$NAME}` and `{$NAME:default}` are substituted when the file is loaded, + * `{env.NAME}` when a request is served. Both read Caddy's environment, which + * no longer holds these three. + */ +function substituteEnvironment(code: string, values: LegacyCaddyValues): string { + const known: Record = { + DOMAIN: values.domain, + ADMIN_DOMAIN: values.adminDomain, + ACTIVITYPUB_TARGET: values.activitypub, + }; + return code + .replace(/\{\$([A-Za-z_][A-Za-z0-9_]*)(?::([^}]*))?\}/g, (whole, name: string, fallback) => + name in known ? known[name] || (fallback ?? '') : whole, + ) + .replace(/\{env\.([A-Za-z_][A-Za-z0-9_]*)\}/g, (whole, name: string) => + name in known ? known[name]! : whole, + ); +} + +const ALIAS = new RegExp( + `(^|[\\s/])(${ALIASED.map((name) => name.replace('-', '\\-')).join('|')}):(\\d+)(?=$|[\\s/])`, + 'g', +); + +/** + * An import of a shipped snippet gains its absolute path and arguments; any + * other relative import is made absolute, since it now sits in caddy/sites/ + * rather than beside the Caddyfile it was relative to. Named snippets the + * file defines are left alone. + */ +function rewriteImport( + code: string, + values: LegacyCaddyValues, + named: ReadonlySet, + changes: Set, +): string { + const match = /^(\s*)import(\s+)(\S+)(.*)$/.exec(code); + if (match === null) { + return code; + } + const [, indent, space, path, rest] = match as unknown as [ + string, + string, + string, + string, + string, + ]; + if (named.has(path)) { + return code; + } + const snippet = /^(?:\.\/|\/etc\/caddy\/)?snippets\/([A-Za-z]+)$/.exec(path); + if (snippet !== null) { + const name = snippet[1]!; + const args = snippetArguments(name, values); + if (args !== null) { + if (rest.trim() !== '') { + throw new UntranslatableCaddyfile( + `caddy/Caddyfile imports snippets/${name} with arguments (${rest.trim()}); on main it took none,\n` + + ' so it cannot be told what they were meant to be.', + ); + } + changes.add('snippet imports given their absolute paths and arguments'); + return `${indent}import${space}/etc/caddy/snippets/${name}${args}`; + } + } + if (path.startsWith('/')) { + return code; + } + changes.add('relative imports made absolute'); + return `${indent}import${space}/etc/caddy/${path.replace(/^\.\//, '')}${rest}`; +} + +/** Does this line's code open a block, close one, or neither? */ +function braces(code: string): { opens: boolean; closes: boolean } { + const tokens = code.trim().split(/\s+/); + return { + opens: tokens.at(-1) === '{', + closes: tokens[0] === '}', + }; +} + +/** + * The operator's Caddyfile, as routes for this layout. Throws + * UntranslatableCaddyfile for what it cannot carry over; Caddy validates the + * rest before anything is written. + */ +export function translateCaddyfile(text: string, values: LegacyCaddyValues): TranslatedCaddyfile { + if (/<<[A-Za-z]/.test(text)) { + throw new UntranslatableCaddyfile( + 'caddy/Caddyfile uses a heredoc, which this migration does not translate', + ); + } + const lines = text.replace(/\r\n/g, '\n').split('\n'); + const named = new Set( + lines + .map((line) => /^\s*\(([^)\s]+)\)\s*\{\s*$/.exec(splitComment(line).code)?.[1]) + .filter((name): name is string => name !== undefined), + ); + const changes = new Set(); + + const site: string[] = []; + const global: string[] = []; + let depth = 0; + /** In the global options block, which is the first block and has no address. */ + let inGlobal = false; + let seenBlock = false; + for (const line of lines) { + const { code: original, comment } = splitComment(line); + let code = substituteEnvironment(original, values); + if (code !== original) { + changes.add( + '{$DOMAIN}, {$ADMIN_DOMAIN} and {$ACTIVITYPUB_TARGET} replaced by their values', + ); + } + const aliased = code.replace( + ALIAS, + (_, before: string, service: string, port: string) => + `${before}${service}-${values.project}:${port}`, + ); + if (aliased !== code) { + changes.add('bare service upstreams pointed at the site’s unique aliases'); + code = aliased; + } + code = rewriteImport(code, values, named, changes); + const { opens, closes } = braces(code); + + if (depth === 0 && opens) { + // Only a block written with no address is global options: one + // whose address is a variable that turned out empty is a mistake. + if (original.trim() === '{' && !seenBlock) { + inGlobal = true; + seenBlock = true; + depth += 1; + changes.add(`the global options block moved to ${GLOBAL_FILE}`); + continue; + } + const addresses = code + .trim() + .slice(0, -1) + .trim() + .split(/\s*,\s*|\s+/); + if (addresses.some((address) => address === '')) { + throw new UntranslatableCaddyfile( + `caddy/Caddyfile has a site block with an empty address once DOMAIN and ADMIN_DOMAIN are\n` + + ` filled in (${original.trim()}). Set ADMIN_DOMAIN in .env, or remove that block.`, + ); + } + } + if (closes) { + depth -= 1; + if (depth < 0) { + throw new UntranslatableCaddyfile( + 'caddy/Caddyfile closes more blocks than it opens', + ); + } + if (depth === 0 && inGlobal) { + inGlobal = false; + continue; + } + } + if (opens) { + depth += 1; + seenBlock = true; + } + if (inGlobal) { + // One level shallower: the stack's Caddyfile supplies the braces. + global.push(`${code}${comment}`.replace(/^\t/, '')); + } else { + site.push(`${code}${comment}`); + } + } + if (depth !== 0) { + throw new UntranslatableCaddyfile('caddy/Caddyfile opens more blocks than it closes'); + } + + const header = [ + `# Routes for ${values.project}, carried over from caddy/Caddyfile, which is kept as`, + '# caddy/Caddyfile.local. This file is yours: edit it, then reload Caddy:', + '#', + '# docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile', + '#', + '# docs/caddy.md lists the snippets and the arguments each one takes.', + '', + ]; + return { + site: `${[...header, ...site].join('\n').trimEnd()}\n`, + global: + global.length === 0 + ? null + : `# Global options carried over from caddy/Caddyfile.\n${global.join('\n').trimEnd()}\n`, + changes: [...changes], + }; +} diff --git a/manager/src/legacy/config.ts b/manager/src/legacy/config.ts new file mode 100644 index 00000000..76e4fe35 --- /dev/null +++ b/manager/src/legacy/config.ts @@ -0,0 +1,111 @@ +// The released `main` layout's single `.env`, split into this layout's `.env` +// and `ghost.env` (docs/install.md#moving-from-the-released-main-layout). +// +// On `main`, `.env` was both Compose's interpolation source and the ghost +// service's env_file, so every key in it reached Ghost. Here operator settings +// stay in `.env`, which Ghost never sees, and Ghost's own configuration moves +// to `ghost.env`. Which is which is decided the way `config validate` decides +// it (config.ts): a key Compose interpolates, or an example documents, is an +// operator setting; anything else was there for Ghost. +import * as env from '../env.ts'; +import { CliError } from '../errors.ts'; +import { isContainerOwned } from '../import/config.ts'; + +/** + * Keys `main` read that this layout replaces: URL and ADMIN_URL, and the + * ActivityPub upstream now written into the Caddy route. Kept only when an + * override of the operator's still interpolates them. + */ +export const SUPERSEDED = ['DOMAIN', 'ADMIN_DOMAIN', 'ACTIVITYPUB_TARGET'] as const; + +/** The old `.env`, read as Compose read it. */ +export interface LegacyEnv { + readonly values: Readonly>; + /** Every key, in the order of its first assignment. */ + readonly keys: readonly string[]; +} + +export function readLegacyEnv(text: string): LegacyEnv { + const multiline = env.scan(text).filter((assignment) => assignment.quoting === 'multiline'); + if (multiline.length > 0) { + throw new CliError( + `.env has values spanning several lines (${multiline.map(({ key }) => key).join(', ')}), which this\n` + + ' migration does not carry over. Put each on one line, then run it again. Nothing has been changed.', + ); + } + return { values: env.toRecord(text), keys: env.keys(text) }; +} + +export interface SplitInput { + readonly legacy: LegacyEnv; + /** Keys this migration writes itself, from what it has worked out. */ + readonly generated: ReadonlySet; + /** Is the key an operator setting, in the old layout or this one? */ + readonly isOperatorKey: (key: string) => boolean; + /** Does this layout's compose.yml, with the operator's overrides, still interpolate it? */ + readonly isInterpolated: (key: string) => boolean; + /** What this layout's ghost service sets itself, by key. */ + readonly container: ReadonlySet; + /** + * What Ghost actually received for each key on `main`, as Compose resolved + * it: env_file values are interpolated, so this, not the text, is the value. + */ + readonly received: Readonly>; +} + +export interface SplitEnv { + /** Operator settings carried into `.env`, besides the generated ones. */ + readonly operator: [string, string][]; + /** Ghost configuration for `ghost.env`. */ + readonly ghost: [string, string][]; + /** Keys left out, with why. Names only: values may be credentials. */ + readonly dropped: { key: string; reason: string }[]; +} + +export function splitLegacyEnv({ + legacy, + generated, + isOperatorKey, + isInterpolated, + container, + received, +}: SplitInput): SplitEnv { + const operator: [string, string][] = []; + const ghost: [string, string][] = []; + const dropped: { key: string; reason: string }[] = []; + for (const key of legacy.keys) { + const value = legacy.values[key]!; + if (generated.has(key)) { + continue; + } + if ((SUPERSEDED as readonly string[]).includes(key) && !isInterpolated(key)) { + dropped.push({ key, reason: 'replaced by URL, ADMIN_URL and the Caddy routes' }); + } else if (key.startsWith('COMPOSE_') || isOperatorKey(key)) { + operator.push([key, value]); + } else if (isContainerOwned(key, container)) { + dropped.push({ + key, + reason: 'set by the container, which took precedence on main too', + }); + } else { + ghost.push([key, received[key] ?? value]); + } + } + return { operator, ghost, dropped }; +} + +/** + * Operator settings whose value Compose would interpolate: a `$` that is not + * doubled. What Compose made of it is not knowable from the text alone, so the + * operator says what was meant, rather than this migration guessing. + */ +export function refuseInterpolated(text: string, operatorKeys: ReadonlySet): void { + const keys = env.lint(text).filter((key) => operatorKeys.has(key)); + if (keys.length > 0) { + throw new CliError( + `.env has values with an unescaped $ (${keys.join(', ')}), which Compose interpolates, so what\n` + + ' they hold cannot be carried over exactly. Write each literal $ as $$, check the site still\n' + + ' starts (docker compose up -d), then run this again. Nothing has been changed.', + ); + } +} diff --git a/manager/src/recovery.ts b/manager/src/recovery.ts index 0128e355..85bf75c9 100644 --- a/manager/src/recovery.ts +++ b/manager/src/recovery.ts @@ -13,7 +13,7 @@ import { copyPresent, sameTree } from './fs.ts'; import { checkRows, loadFile } from './import/database.ts'; import type { Io } from './io.ts'; import { ok } from './report.ts'; -import { DATA_DIRS, readSettings } from './site.ts'; +import { DATA_DIRS, readSettings, UPDATE_DIR } from './site.ts'; // --- The services ----------------------------------------------------------------- @@ -60,6 +60,47 @@ export function describeServices(running: string[] | null): string { return `These of its services are still running: ${running.join(', ')}.`; } +// --- Putting files back --------------------------------------------------------- + +/** + * Copies of the files an operation may change, kept in UPDATE_DIR, to put + * back if it fails before it may no longer: self-update and the migration + * from the released main layout. + */ +export class Snapshot { + readonly dir: string; + readonly root: string; + readonly paths: string[]; + + constructor(dir: string, paths: string[]) { + this.dir = dir; + this.root = join(dir, UPDATE_DIR); + this.paths = paths; + } + + take(): void { + mkdirSync(join(this.root, 'files'), { recursive: true, mode: 0o700 }); + copyPresent(this.dir, this.paths, join(this.root, 'files')); + } + + /** Every path as it was: copied back, or removed when it did not exist. */ + restore(): void { + for (const path of this.paths) { + const target = join(this.dir, path); + const kept = join(this.root, 'files', path); + rmSync(target, { recursive: true, force: true }); + if (existsSync(kept)) { + mkdirSync(dirname(target), { recursive: true, mode: 0o755 }); + cpSync(kept, target, { recursive: true, preserveTimestamps: true }); + } + } + } + + remove(): void { + rmSync(this.root, { recursive: true, force: true }); + } +} + // --- Setting a site aside ------------------------------------------------------- /** diff --git a/manager/test/legacy.test.ts b/manager/test/legacy.test.ts new file mode 100644 index 00000000..415904d2 --- /dev/null +++ b/manager/test/legacy.test.ts @@ -0,0 +1,254 @@ +// Migration 0001-compose-profiles: what it makes of the released main +// layout's Caddyfile and `.env`. What Caddy and Compose make of the result is +// tests/e2e/migrate-main.sh. +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, test } from 'node:test'; +import { + translateCaddyfile, + UntranslatableCaddyfile, + type LegacyCaddyValues, +} from '../src/legacy/caddy.ts'; +import { readLegacyEnv, refuseInterpolated, splitLegacyEnv } from '../src/legacy/config.ts'; +import { REPO } from './site.ts'; + +const MAIN = join(REPO, 'tests', 'fixtures', 'released-main'); +const EXAMPLE = readFileSync(join(MAIN, 'caddy', 'Caddyfile.example'), 'utf8'); + +const VALUES: LegacyCaddyValues = { + project: 'ghost-docker', + domain: 'example.com', + adminDomain: '', + activitypub: 'https://ap.ghost.org', +}; +const translate = (text: string, values: Partial = {}) => + translateCaddyfile(text, { ...VALUES, ...values }); + +/** The lines that are not comments: what Caddy loads. */ +const code = (text: string) => + text + .split('\n') + .filter((line) => line.trim() !== '' && !line.trim().startsWith('#')) + .join('\n'); + +/** Uncomment the example's block that starts with `first`. */ +function uncomment(text: string, first: string): string { + const lines = text.split('\n'); + let inside = false; + return lines + .map((line) => { + if (line.startsWith(`# ${first}`)) { + inside = true; + } + if (inside && /^#( |$)/.test(line)) { + const uncommented = line.replace(/^# ?/, ''); + if (uncommented === '}') { + inside = false; + } + return uncommented; + } + return line; + }) + .join('\n'); +} + +describe('the Caddyfile', () => { + test('the example, as most sites have it, becomes routes with arguments and aliases', () => { + const { site, global, changes } = translate(EXAMPLE); + const loaded = code(site); + assert.match(loaded, /^example\.com \{$/m); + assert.match(loaded, /^\timport \/etc\/caddy\/snippets\/Logging$/m); + assert.match( + loaded, + /^\timport \/etc\/caddy\/snippets\/TrafficAnalytics traffic-analytics-ghost-docker:3000$/m, + ); + assert.match( + loaded, + /^\timport \/etc\/caddy\/snippets\/ActivityPub https:\/\/ap\.ghost\.org$/m, + ); + assert.match(loaded, /^\timport \/etc\/caddy\/snippets\/SecurityHeaders ""$/m); + assert.match(loaded, /^\t\treverse_proxy ghost-ghost-docker:2368$/m); + assert.doesNotMatch(loaded, /\{\$|import snippets\/|ghost:2368/); + assert.equal(global, null); + assert.ok(changes.length > 0); + }); + + test("the admin domain's block is filled in, and may frame the site", () => { + const { site } = translate(uncomment(EXAMPLE, '{$ADMIN_DOMAIN}'), { + adminDomain: 'admin.example.com', + }); + assert.match(code(site), /^admin\.example\.com \{$/m); + assert.equal(code(site).match(/SecurityHeaders "admin\.example\.com"/g)?.length, 2); + }); + + test('an admin block with no ADMIN_DOMAIN is refused, not turned into global options', () => { + assert.throws( + () => translate(uncomment(EXAMPLE, '{$ADMIN_DOMAIN}')), + (error) => + error instanceof UntranslatableCaddyfile && /empty address/.test(error.message), + ); + }); + + test('the www redirect keeps redirecting to the domain', () => { + const { site } = translate(uncomment(EXAMPLE, 'www.{$DOMAIN}')); + assert.match(code(site), /^www\.example\.com \{$/m); + assert.match(code(site), /^\tredir https:\/\/example\.com\{uri\}$/m); + }); + + test("the site's own ActivityPub is addressed by its alias", () => { + const { site } = translate(EXAMPLE, { activitypub: 'activitypub-ghost-docker:8080' }); + assert.match(code(site), /snippets\/ActivityPub activitypub-ghost-docker:8080$/m); + // Written straight into a route, the bare name is rewritten too. + const direct = translate('example.com {\n\treverse_proxy http://activitypub:8080\n}\n'); + assert.match(direct.site, /reverse_proxy http:\/\/activitypub-ghost-docker:8080/); + }); + + test('custom routes are kept as written', () => { + const custom = EXAMPLE.replace( + '\t# Default proxy everything else to Ghost', + '\tredir /old /new 301\n\thandle /status {\n\t\trespond "ok"\n\t}\n\n\t# Default proxy everything else to Ghost', + ).concat('\nstatus.example.com {\n\treverse_proxy 172.17.0.1:9000\n}\n'); + const { site } = translate(custom); + assert.match(site, /^\tredir \/old \/new 301$/m); + assert.match(site, /^\thandle \/status \{\n\t\trespond "ok"\n\t\}$/m); + assert.match(site, /^status\.example\.com \{\n\treverse_proxy 172\.17\.0\.1:9000\n\}$/m); + }); + + test('a global options block moves to caddy/global, one level shallower', () => { + const { site, global } = translate(`{\n\temail ops@example.com\n\tdebug\n}\n\n${EXAMPLE}`); + assert.equal( + global, + '# Global options carried over from caddy/Caddyfile.\nemail ops@example.com\ndebug\n', + ); + assert.doesNotMatch(code(site), /email ops/); + assert.match(code(site), /^example\.com \{$/m); + }); + + test('named snippets stay named; other relative imports become absolute', () => { + const { site } = translate( + '(common) {\n\tencode gzip\n}\n\nexample.com {\n\timport common\n\timport extra/*.caddy\n\treverse_proxy ghost:2368\n}\n', + ); + assert.match(site, /^\timport common$/m); + assert.match(site, /^\timport \/etc\/caddy\/extra\/\*\.caddy$/m); + }); + + test('runtime placeholders and other variables are left alone', () => { + const { site } = translate( + 'example.com {\n\theader X-Host {env.DOMAIN}\n\theader X-Other {$OTHER} {uri}\n}\n', + ); + assert.match(site, /X-Host example\.com$/m); + assert.match(site, /X-Other \{\$OTHER\} \{uri\}$/m); + }); + + test('a snippet imported with arguments it never took is refused', () => { + assert.throws( + () => translate('example.com {\n\timport snippets/Logging verbose\n}\n'), + UntranslatableCaddyfile, + ); + }); + + test('unbalanced braces are refused', () => { + assert.throws(() => translate('example.com {\n\trespond ok\n'), UntranslatableCaddyfile); + assert.throws(() => translate('}\n'), UntranslatableCaddyfile); + }); + + test('comments are kept as they were', () => { + const { site } = translate(EXAMPLE); + assert.match(site, /^# \{\$ADMIN_DOMAIN\} \{$/m); + assert.match(site, /^\t# Optional: Enable gzip compression$/m); + }); +}); + +describe('the .env', () => { + const OLD = [ + 'COMPOSE_PROFILES=analytics', + 'DOMAIN=example.com', + 'ADMIN_DOMAIN=', + 'HTTP_PORT=80', + 'DATABASE_ROOT_PASSWORD=root', + 'DATABASE_PASSWORD=app', + 'TINYBIRD_ADMIN_TOKEN=p.token', + 'TINYBIRD_SYNC_AUTH=shared', + 'mail__transport=SMTP', + 'mail__options__auth__pass="p$$ss"', + 'mail__from="\'Acme\' "', + 'url=https://ignored.example.com', + 'database__connection__host=elsewhere', + 'TZ=Europe/London', + 'GHOST_VERSION=6-alpine', + '', + ].join('\n'); + const OPERATOR = new Set([ + 'DOMAIN', + 'ADMIN_DOMAIN', + 'HTTP_PORT', + 'DATABASE_ROOT_PASSWORD', + 'DATABASE_PASSWORD', + 'TINYBIRD_ADMIN_TOKEN', + 'TINYBIRD_SYNC_AUTH', + 'GHOST_VERSION', + ]); + const split = (interpolated: string[] = []) => + splitLegacyEnv({ + legacy: readLegacyEnv(OLD), + generated: new Set([ + 'COMPOSE_PROFILES', + 'HTTP_PORT', + 'DATABASE_ROOT_PASSWORD', + 'DATABASE_PASSWORD', + 'GHOST_VERSION', + ]), + isOperatorKey: (key) => key.startsWith('COMPOSE_') || OPERATOR.has(key), + isInterpolated: (key) => interpolated.includes(key), + container: new Set(['url', 'admin__url', 'NODE_ENV']), + received: { mail__options__auth__pass: 'p$ss', mail__transport: 'SMTP' }, + }); + + test("Ghost's configuration moves to ghost.env, operator settings stay", () => { + const { operator, ghost, dropped } = split(); + assert.deepEqual(operator, [ + ['TINYBIRD_ADMIN_TOKEN', 'p.token'], + ['TINYBIRD_SYNC_AUTH', 'shared'], + ]); + assert.deepEqual(ghost, [ + ['mail__transport', 'SMTP'], + ['mail__options__auth__pass', 'p$ss'], + ['mail__from', "'Acme' "], + // Everything in .env reached Ghost on main, so a setting of the + // container's own goes with it. + ['TZ', 'Europe/London'], + ]); + assert.deepEqual( + dropped.map(({ key }) => key), + ['DOMAIN', 'ADMIN_DOMAIN', 'url', 'database__connection__host'], + ); + }); + + test('DOMAIN stays when an override still uses it', () => { + const { operator, dropped } = split(['DOMAIN']); + assert.deepEqual(operator[0], ['DOMAIN', 'example.com']); + assert.ok(!dropped.some(({ key }) => key === 'DOMAIN')); + }); + + test('ghost.env gets what Ghost received, which Compose interpolated', () => { + const { ghost } = split(); + assert.deepEqual( + ghost.find(([key]) => key === 'mail__options__auth__pass'), + ['mail__options__auth__pass', 'p$ss'], + ); + }); + + test('an operator value Compose would interpolate is refused', () => { + assert.throws( + () => refuseInterpolated('DATABASE_PASSWORD=pa$word\n', new Set(['DATABASE_PASSWORD'])), + /unescaped \$ \(DATABASE_PASSWORD\)/, + ); + // Ghost's own are carried as Ghost received them. + refuseInterpolated('mail__options__auth__pass=pa$word\n', new Set()); + }); + + test('a value spanning lines is refused rather than dropped', () => { + assert.throws(() => readLegacyEnv('KEY="one\ntwo"\n'), /KEY/); + }); +}); diff --git a/manager/test/migrate-main.test.ts b/manager/test/migrate-main.test.ts new file mode 100644 index 00000000..0c891748 --- /dev/null +++ b/manager/test/migrate-main.test.ts @@ -0,0 +1,383 @@ +// `self-update` in an installation of the released main layout: migration +// 0001-compose-profiles, against a scripted daemon, Compose and git. What it +// does on a real host is tests/e2e/migrate-main.sh; this is what it decides. +import assert from 'node:assert/strict'; +import { cpSync, existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { basename, join } from 'node:path'; +import { afterEach, beforeEach, describe, test } from 'node:test'; +import * as env from '../src/env.ts'; +import { failed, harness, json, ok, type Harness, type ProgramResult } from './helpers.ts'; +import { + imageApi, + imageStack, + REFERENCE, + REPO, + resolvedProject, + scriptSite, + writeSiteData, + type ScriptedSite, +} from './site.ts'; + +const MAIN = join(REPO, 'tests', 'fixtures', 'released-main'); +/** The Ghost-CLI-layout image main's site runs. */ +const OLD_GHOST = `sha256:${'5'.repeat(64)}`; +const CADDY = `caddy:2.10.2-alpine@sha256:${'6'.repeat(64)}`; +const DB = `mysql:8.0.44@sha256:${'4'.repeat(64)}`; + +const OLD_ENV = [ + '# Use the below flags to enable the Analytics or ActivityPub containers as well', + 'COMPOSE_PROFILES=activitypub', + 'DOMAIN=example.com', + 'HTTP_PORT=80', + 'HTTPS_PORT=443', + 'DATABASE_ROOT_PASSWORD=reallysecurerootpassword', + 'DATABASE_PASSWORD=ghostpassword', + 'ACTIVITYPUB_TARGET=activitypub:8080', + 'mail__transport=SMTP', + 'mail__options__host=smtp.example.com', + 'mail__options__auth__pass="pa$$word"', + 'mail__from="\'Acme Support\' "', + 'labs__publicAPI=true', + 'UPLOAD_LOCATION=./data/ghost', + 'MYSQL_DATA_LOCATION=./data/mysql', + '', +].join('\n'); + +let h: Harness; +let site: ScriptedSite; +let project: string; +/** Files git says the operator changed, and files only local commits changed. */ +let edited: string[]; +let committed: string[]; +/** What `caddy validate` answers. */ +let caddyValidates: { status: number; stderr?: string }; +const validated: string[] = []; + +const readSite = (file: string) => readFileSync(join(h.dir, file), 'utf8'); +const envOf = (file: string) => env.toRecord(readSite(file)); +const migrate = (...args: string[]) => h.run('self-update', ...args); + +/** Every file outside the data, to compare before and after. */ +function files(dir = h.dir, prefix = ''): Record { + const found: Record = {}; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (prefix === '' && ['data', 'backups'].includes(entry.name)) { + continue; + } + const path = join(dir, entry.name); + if (entry.isDirectory()) { + Object.assign(found, files(path, `${prefix}${entry.name}/`)); + } else { + found[`${prefix}${entry.name}`] = readFileSync(path, 'utf8'); + } + } + return found; +} + +/** `docker compose config`: main's layout with the site's `.env`, or this one's with a staged one. */ +function config(_args: string[], envFile?: string): ProgramResult { + const values = env.toRecord(readFileSync(envFile ?? join(h.dir, '.env'), 'utf8')); + const legacy = values.URL === undefined; + const environment: Record = legacy + ? // main's env_file is the whole of .env. + { ...values, url: `https://${values.DOMAIN}`, NODE_ENV: 'production' } + : { + ...env.toRecord(existsSync(join(h.dir, 'ghost.env')) ? readSite('ghost.env') : ''), + NODE_ENV: 'production', + url: values.URL!, + admin__url: values.ADMIN_URL ?? '', + server__host: '0.0.0.0', + server__port: '2368', + paths__contentPath: values.GHOST_CONTENT_PATH ?? '', + database__client: 'mysql', + }; + const resolved = JSON.parse( + resolvedProject( + h.dir, + { ghost: legacy ? 'ghost:6-alpine' : values.GHOST_IMAGE_REF!, db: DB, caddy: CADDY }, + {}, + envFile, + ), + ); + resolved.services.ghost.environment = Object.fromEntries( + Object.entries(environment).map(([key, value]) => [key, value.replaceAll('$', '$$')]), + ); + if (legacy) { + resolved.services.ghost.volumes[0].target = '/var/lib/ghost/content'; + } + return ok(JSON.stringify(resolved)); +} + +beforeEach(() => { + h = harness(); + imageStack(h); + h.env.GD_VERSION_FILE = join(h.dir, '..', `${basename(h.dir)}-version.json`); + writeFileSync( + h.env.GD_VERSION_FILE, + JSON.stringify({ version: 'v0.2.0', commit: 'c'.repeat(40) }), + ); + h.env.GD_CHANNEL = 'stable'; + + // A clone of main, set up as its README said. + cpSync(MAIN, h.dir, { recursive: true }); + writeFileSync(join(h.dir, '.env'), OLD_ENV); + cpSync(join(MAIN, 'caddy', 'Caddyfile.example'), join(h.dir, 'caddy', 'Caddyfile')); + writeSiteData(h.dir); + project = basename(h.dir); + edited = []; + committed = []; + caddyValidates = { status: 0 }; + validated.length = 0; + + const images = imageApi({ ghost: { '6.67.0-next-alpine': '6.67.0' } }); + h.daemon.api = (request) => { + if ( + request.method === 'GET' && + decodeURIComponent(request.path) === `/images/${OLD_GHOST}/json` + ) { + return json(200, { + Id: OLD_GHOST, + RepoDigests: [`ghost@sha256:${'7'.repeat(64)}`], + Config: { + Env: [ + 'GHOST_VERSION=6.67.0', + 'GHOST_INSTALL=/var/lib/ghost', + 'GHOST_CONTENT=/var/lib/ghost/content', + 'GHOST_CLI_INSTALL=/usr/local/lib/ghost-cli', + ], + }, + }); + } + return images(request); + }; + const container = (service: string, image: string, imageId: string) => ({ + Id: `${service}-id`, + Names: [`/${project}-${service}-1`], + Image: image, + ImageID: imageId, + State: 'running', + Status: 'Up', + Labels: { + 'com.docker.compose.project': project, + 'com.docker.compose.project.working_dir': h.dir, + 'com.docker.compose.service': service, + }, + }); + h.daemon.containers = [ + container('ghost', 'ghost:6-alpine', OLD_GHOST), + container('db', DB, `sha256:${'4'.repeat(64)}`), + container('caddy', CADDY, `sha256:${'6'.repeat(64)}`), + ]; + h.daemon.gitRun = (args) => { + const command = args[args.indexOf(h.dir) + 1]; + const paths = args.slice(args.indexOf('--') + 1); + switch (command) { + case 'rev-parse': + return ok(`${h.dir}\n`); + case 'ls-files': + return ok(paths.filter((path) => existsSync(join(MAIN, path))).join('\n')); + case 'status': + return ok(edited.map((file) => ` M ${file}\n`).join('')); + case 'for-each-ref': + return ok('refs/remotes/origin/main\n'); + case 'log': + return ok(committed.join('\n')); + default: + return failed(1, `unexpected: git ${args.join(' ')}`); + } + }; + + site = scriptSite(h, config, { ghost: REFERENCE, db: DB, caddy: CADDY }); + site.running.add('ghost').add('db').add('caddy'); + site.onUp = () => site.running.add('caddy'); + const scripted = h.daemon.run!; + h.daemon.run = (spec) => { + if (spec.entrypoint[0] === 'caddy') { + const caddy = spec.binds[0]!.split(':')[0]!; + validated.push(readFileSync(join(caddy, 'sites', 'site.caddy'), 'utf8')); + return caddyValidates; + } + return scripted(spec); + }; +}); +afterEach(() => h.cleanup()); + +describe('migrating the released main layout', () => { + test('moves the site onto this layout, keeping its project, credentials and Ghost', async () => { + const result = await migrate(); + assert.equal(result.code, 0, result.stderr); + assert.match(result.stdout, /migration 0001-compose-profiles/); + + const settings = envOf('.env'); + assert.equal(settings.COMPOSE_PROFILES, 'production,activitypub'); + assert.equal(settings.SITE_MODE, 'production'); + assert.equal(settings.COMPOSE_PROJECT_NAME, project); + assert.equal(settings.PROJECT_DIR, h.dir); + assert.equal(settings.URL, 'https://example.com'); + assert.equal(settings.GHOST_IMAGE_REF, REFERENCE); + assert.equal(settings.GHOST_VERSION, '6.67.0-next-alpine'); + assert.equal(settings.DATABASE_PASSWORD, 'ghostpassword'); + assert.equal(settings.DATABASE_ROOT_PASSWORD, 'reallysecurerootpassword'); + for (const key of ['DOMAIN', 'ACTIVITYPUB_TARGET', 'mail__transport']) { + assert.equal(settings[key], undefined, key); + } + + const ghost = envOf('ghost.env'); + assert.equal(ghost.mail__transport, 'SMTP'); + assert.equal(ghost.mail__options__auth__pass, 'pa$word'); + assert.equal(ghost.labs__publicAPI, 'true'); + assert.equal(ghost.DATABASE_PASSWORD, undefined); + + const routes = readSite('caddy/sites/site.caddy'); + assert.match(routes, /^example\.com \{$/m); + assert.match(routes, new RegExp(`snippets/ActivityPub activitypub-${project}:8080$`, 'm')); + assert.match(routes, new RegExp(`reverse_proxy ghost-${project}:2368$`, 'm')); + assert.equal( + readSite('caddy/Caddyfile.local'), + readFileSync(join(MAIN, 'caddy', 'Caddyfile.example'), 'utf8'), + ); + assert.equal( + readSite('caddy/Caddyfile'), + readFileSync(join(REPO, 'caddy', 'Caddyfile'), 'utf8'), + ); + assert.equal(readSite('compose.yml'), readFileSync(join(REPO, 'compose.yml'), 'utf8')); + assert.equal(validated.length, 1); + + const metadata = JSON.parse(readSite('.ghost-docker.json')); + assert.equal(metadata.source, 'image'); + assert.equal(metadata.stack.version, 'v0.2.0'); + assert.equal(metadata.site.project, project); + assert.equal(metadata.ghost.version, '6.67.0'); + assert.deepEqual(metadata.profiles, ['production', 'activitypub']); + assert.ok('compose.yml' in metadata.payload && 'ghost-docker' in metadata.payload); + assert.match(readSite('ghost-docker'), /^readonly GD_PINNED_IMAGE=".+"$/m); + + assert.equal(readdirSync(join(h.dir, 'backups')).length, 1); + assert.ok(!existsSync(join(h.dir, '.ghost-docker-update'))); + assert.ok(!existsSync(join(h.dir, '.ghost-docker.lock'))); + assert.deepEqual([...site.running].sort(), ['caddy', 'db', 'ghost']); + }); + + test('ActivityPub stays where main sent it: the hosted service unless the target said otherwise', async () => { + writeFileSync( + join(h.dir, '.env'), + OLD_ENV.replace('ACTIVITYPUB_TARGET=activitypub:8080\n', ''), + ); + const result = await migrate(); + assert.equal(result.code, 0, result.stderr); + assert.match( + readSite('caddy/sites/site.caddy'), + /snippets\/ActivityPub https:\/\/ap\.ghost\.org$/m, + ); + assert.equal(envOf('.env').COMPOSE_PROFILES, 'production,activitypub'); + }); + + test('a second run is an ordinary self-update, and finds nothing to do', async () => { + assert.equal((await migrate()).code, 0); + const after = files(); + const again = await migrate(); + assert.equal(again.code, 0, again.stderr); + assert.match(again.stdout, /already runs v0\.2\.0/); + assert.deepEqual(files(), after); + }); + + test('--check says what it would do and changes nothing', async () => { + const before = files(); + const result = await migrate('--check'); + assert.equal(result.code, 0, result.stderr); + assert.match(result.stdout, /would move this site onto v0\.2\.0/); + assert.match(result.stdout, /DOMAIN not carried/); + assert.deepEqual(files(), before); + assert.equal(site.compose.filter((args) => args[0] === 'stop').length, 0); + }); + + test('an edited stack file stops it before anything changes', async () => { + edited = ['compose.yml']; + committed = ['caddy/snippets/Logging']; + const before = files(); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /caddy\/snippets\/Logging\n\s+compose\.yml/); + assert.match(result.stderr, /Nothing has been changed/); + assert.deepEqual(files(), before); + }); + + test('Ghost below the first next image stops it, saying to upgrade Ghost first', async () => { + const api = h.daemon.api!; + h.daemon.api = (request) => + decodeURIComponent(request.path) === `/images/${OLD_GHOST}/json` + ? json(200, { + Id: OLD_GHOST, + RepoDigests: [`ghost@sha256:${'7'.repeat(64)}`], + Config: { Env: ['GHOST_VERSION=6.40.0'] }, + }) + : api(request); + const before = files(); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /runs Ghost 6\.40\.0.*start at Ghost 6\.61\.0/s); + assert.deepEqual(files(), before); + }); + + test('routes Caddy will not load stop it before anything changes', async () => { + caddyValidates = { + status: 1, + stderr: 'Error: adapting config: unknown directive: frobnicate', + }; + const before = files(); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /unknown directive: frobnicate/); + assert.deepEqual(files(), before); + }); + + test('a failure before startup puts the files back and starts Ghost again', async () => { + const before = files(); + // The checked backup fails: its dump does not load. + const scripted = h.daemon.run!; + h.daemon.run = (spec) => + spec.entrypoint[0] === 'sh' + ? { status: 4, stderr: 'the dump of ghost does not load' } + : scripted(spec); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /Restored: the site is on the released main layout again/); + assert.deepEqual(files(), before); + assert.ok(site.running.has('ghost')); + assert.equal( + site.compose.filter((args) => args[0] === 'up' && !args.includes('--no-recreate')) + .length, + 0, + ); + }); + + test('a failure at startup stops the site, puts the files back and leaves the data', async () => { + const before = files(); + site.ups.push(failed(1, 'container ghost is unhealthy')); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /The site needs you/); + assert.match(result.stderr, /Nothing was loaded over them/); + assert.match(result.stderr, /docker compose up -d/); + assert.match(result.stderr, /restore --yes backups\//); + // The snapshot stays for the operator; everything else is main's again. + const after = files(); + assert.deepEqual( + Object.fromEntries( + Object.entries(after).filter(([path]) => !path.startsWith('.ghost-docker-update/')), + ), + before, + ); + assert.equal( + readFileSync(join(h.dir, 'data', 'ghost', 'images', 'photo.jpg'), 'utf8'), + 'jpeg', + ); + }); + + test('a site that is not running is refused: its Ghost version is unknown', async () => { + h.daemon.containers = []; + const result = await migrate(); + assert.equal(result.code, 1); + assert.match(result.stderr, /must be running/); + }); +}); diff --git a/manager/test/site.ts b/manager/test/site.ts index f1ca47d7..60c63183 100644 --- a/manager/test/site.ts +++ b/manager/test/site.ts @@ -322,7 +322,7 @@ export function writeSiteData(dir: string): void { */ export function scriptSite( h: Harness, - config: (args: string[]) => ProgramResult | undefined, + config: (args: string[], envFile?: string) => ProgramResult | undefined, images: Record = { ghost: REFERENCE, db: `mysql:8.0.44@sha256:${'4'.repeat(64)}`, @@ -362,11 +362,11 @@ export function scriptSite( } return undefined; }; - h.daemon.composeRun = (args, _env, input) => { + h.daemon.composeRun = (args, _env, input, _dir, envFile) => { site.compose.push(args); switch (args[0]) { case 'config': { - const answer = config(args); + const answer = config(args, envFile); if (answer === undefined || answer.exitCode !== 0) { return answer; } diff --git a/tests/e2e/migrate-main.sh b/tests/e2e/migrate-main.sh new file mode 100755 index 00000000..b0661d73 --- /dev/null +++ b/tests/e2e/migrate-main.sh @@ -0,0 +1,288 @@ +#!/usr/bin/env bash +# Migration 0001-compose-profiles, against real containers: an installation +# of the released main layout moved onto this one by the served launcher. +# +# tests/e2e/migrate-main.sh +# +# The installation is made as main's README made one: a git clone of the +# stack (tests/fixtures/released-main, committed, untagged, with a remote- +# tracking branch), `.env` and `caddy/Caddyfile` copied from the examples and +# edited, ActivityPub enabled, a custom route, a global options block, and a +# compose.override.yml. It runs the official `ghost:6-alpine` image, whose +# layout this release does not run. Then: +# +# - an edited stack file, and a release image that cannot be pulled, stop +# the migration before anything changes; +# - Ghost never becoming healthy on the new layout stops the site, puts +# main's files back, and the site starts again on them with its data; +# - the migration through the served launcher, piped as from curl, keeps +# the project, its volumes, credentials, data, Ghost's version, the +# routes and the override, and the site answers through Caddy; +# - running it again is an ordinary self-update with nothing to do. +# +# It pulls images, starts containers and publishes Caddy on two loopback +# ports. Caddy's own CA stands in for a real one (`local_certs`), so nothing +# asks an ACME server. Exits non-zero at the first check that fails. +set -euo pipefail + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd -P) +REGISTRY=ghcr.io/tryghost/ghost-docker +# A version no real release has, so a published image is never mistaken for it. +RELEASE=v0.0.2-beta.1 +IMAGE=$REGISTRY:$RELEASE +DOMAIN=ghost-e2e.test +HTTP_PORT=18080 +HTTPS_PORT=18443 + +# shellcheck source=/dev/null +source "$ROOT/tests/e2e/skip.sh" +require_docker +for tool in git curl; do + command -v "$tool" >/dev/null || { + printf 'migrate-main.sh needs %s\n' "$tool" >&2 + exit 1 + } +done + +WORK=$(mktemp -d "${TMPDIR:-/tmp}/ghost-docker-migrate-e2e.XXXXXXXX") +WORK=$(CDPATH='' cd -- "$WORK" && pwd -P) +S=$WORK/sites/main-site +CURRENT=setup +OUT="" +RC=0 + +cleanup() { + local rc=$? + set +e + [[ -f $S/compose.yml ]] && docker compose --project-directory "$S" -f "$S/compose.yml" \ + --profile '*' down --volumes --remove-orphans --timeout 5 >/dev/null 2>&1 + docker run --rm --user 0 --entrypoint rm -v "$WORK:/work" "$IMAGE" -rf /work/sites >/dev/null 2>&1 + docker rmi "$IMAGE" >/dev/null 2>&1 + case $WORK in */ghost-docker-migrate-e2e.*) rm -rf -- "$WORK" ;; esac + exit "$rc" +} +trap cleanup EXIT + +step() { + CURRENT=$1 + printf '\n== %s\n' "$1" +} +ok() { printf ' ok %s\n' "$1"; } +fail() { + printf '\nFAILED in "%s": %s\n' "$CURRENT" "$1" >&2 + [[ -z ${2:-} ]] || printf -- '--- output ---\n%s\n--------------\n' "$2" >&2 + exit 1 +} +run() { + set +e + OUT=$("$@" 2>&1 /dev/null | grep -q 'Ghost booted'; do + ((waited < 300)) || fail "Ghost did not boot" "$(compose_in logs --tail 40 ghost 2>&1)" + sleep 5 + waited=$((waited + 5)) + done +} +root_sql() { + compose_in exec -T -e MYSQL_PWD=e2e-root-password db mysql -uroot -N -B ghost -e "$1" +} +# https PATH -- through Caddy on the host's port, as a browser would ask. +https() { + curl --silent --insecure --noproxy '*' --max-time 20 --header "Host: $DOMAIN" \ + --resolve "$DOMAIN:$HTTPS_PORT:127.0.0.1" "https://$DOMAIN:$HTTPS_PORT$1" || true +} + +# --- The release, and main's installation ----------------------------------------- + +step "Build the release" +run docker build --quiet --file "$ROOT/manager/Dockerfile" --build-arg "GD_VERSION=$RELEASE" \ + --build-arg "GD_COMMIT=$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || printf '')" --tag "$IMAGE" "$ROOT" +expect_status 0 +ok "$IMAGE" + +step "An installation of the released main layout, as its README made one" +mkdir -p "$S" +cp -R "$ROOT/tests/fixtures/released-main/." "$S/" +rm "$S/README.md" +cp -R "$ROOT/tinybird" "$S/tinybird" +printf '.env\ndata\n' >"$S/.gitignore" +git -C "$S" init --quiet +git -C "$S" add -A +git -C "$S" -c user.name=e2e -c user.email=e2e@example.com commit --quiet -m 'main' +# As a clone has it: the commit is on a remote-tracking branch, and untagged. +git -C "$S" update-ref refs/remotes/origin/main HEAD +cp "$S/.env.example" "$S/.env" +sed -i.bak \ + -e "s/^# COMPOSE_PROFILES=.*/COMPOSE_PROFILES=activitypub/" \ + -e "s/^DOMAIN=.*/DOMAIN=$DOMAIN/" \ + -e "s/^HTTP_PORT=.*/HTTP_PORT=$HTTP_PORT/" \ + -e "s/^HTTPS_PORT=.*/HTTPS_PORT=$HTTPS_PORT/" \ + -e "s/^DATABASE_ROOT_PASSWORD=.*/DATABASE_ROOT_PASSWORD=e2e-root-password/" \ + -e "s/^DATABASE_PASSWORD=.*/DATABASE_PASSWORD=e2e-app-pa\$\$word/" \ + -e "s/^# ACTIVITYPUB_TARGET=.*/ACTIVITYPUB_TARGET=activitypub:8080/" \ + -e "s/^mail__options__auth__pass=.*/mail__options__auth__pass=e2e-smtp-secret/" \ + "$S/.env" +rm "$S/.env.bak" +printf 'labs__publicAPI=true\n' >>"$S/.env" +# The example, with Caddy's own CA, and a route of the operator's own. +{ + printf '{\n\tlocal_certs\n}\n\n' + sed 's|^\t# Default proxy everything else to Ghost|\thandle /e2e-custom {\n\t\trespond "custom route kept"\n\t}\n\n\t# Default proxy everything else to Ghost|' \ + "$S/caddy/Caddyfile.example" +} >"$S/caddy/Caddyfile" +cat >"$S/compose.override.yml" <<'EOF' +services: + ghost: + environment: + e2e__override: kept +EOF +run compose_in up -d +expect_status 0 +wait_for_ghost +project=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$(ghost_id)") +version=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$(ghost_id)" | sed -n 's/^GHOST_VERSION=//p') +root_sql 'CREATE TABLE e2e_marker (note varchar(64)); INSERT INTO e2e_marker VALUES ("kept")' +compose_in exec -T ghost sh -c 'printf kept >/var/lib/ghost/content/images/e2e-marker.txt' +[[ $(https /e2e-custom) == 'custom route kept' ]] || fail "main's custom route does not answer" "$(https /e2e-custom)" +docker volume inspect "${project}_caddy_data" >/dev/null || fail "no ${project}_caddy_data volume" +ok "project $project, Ghost $version on ghost:6-alpine, with ActivityPub, a custom route and an override" + +# --- Refused before anything changes ------------------------------------------------ + +step "An edited stack file stops it before anything changes" +printf '# mine\n' >>"$S/compose.yml" +before=$(fingerprint) +ghost_before=$(ghost_id) +served self-update +expect_status 1 +expect_output 'these files of the stack were changed here' +expect_output 'compose\.yml' +expect_output 'Nothing has been changed' +[[ $(fingerprint) == "$before" && $(ghost_id) == "$ghost_before" ]] || fail "a refused migration changed the site" +git -C "$S" checkout --quiet -- compose.yml +ok "refused, naming compose.yml" + +step "An image the release cannot pull stops it before anything changes" +cp "$S/compose.override.yml" "$WORK/override.yml" +cat >>"$S/compose.override.yml" <<'EOF' + e2e-unpullable: + image: ghcr.io/tryghost/ghost-docker-e2e-does-not-exist:1 + profiles: [production] +EOF +before=$(fingerprint) +served self-update +expect_status 1 +expect_output "images could not be pulled" +expect_output 'Nothing has been changed' +[[ $(fingerprint) == "$before" && $(ghost_id) == "$ghost_before" ]] || fail "a refused migration changed the site" +cp "$WORK/override.yml" "$S/compose.override.yml" +ok "refused while the site kept running" + +step "--check says what it would do and changes nothing" +before=$(fingerprint) +served self-update --check +expect_status 0 +expect_output "would move this site onto $RELEASE" +expect_output "Ghost +$version" +[[ $(fingerprint) == "$before" && $(ghost_id) == "$ghost_before" ]] || fail "--check changed the site" +ok "nothing changed" + +# --- Failing after startup --------------------------------------------------------- + +step "Ghost never healthy on the new layout: stopped, main's files back, data kept" +cat >>"$S/compose.override.yml" <<'EOF' + healthcheck: + test: [CMD, "false"] + interval: 2s + start_period: 0s + start_interval: 2s + retries: 1 +EOF +before=$(fingerprint) +served self-update +expect_status 1 +expect_output 'The site needs you' +expect_output 'Nothing was loaded over them' +expect_output 'docker compose up -d' +[[ $(fingerprint) == "$before" ]] || fail "main's files were not put back" "$(diff <(printf '%s\n' "$before") <(fingerprint) || true)" +[[ -z $(compose_in ps -q) ]] || fail "the migrated site was left running" "$(compose_in ps)" +ok "stopped, with main's files back" + +# The operator's part: the snapshot and the backup are no longer needed. +rm -rf "$S/.ghost-docker-update" "$S/backups" +cp "$WORK/override.yml" "$S/compose.override.yml" +run compose_in up -d +expect_status 0 +wait_for_ghost +[[ $(root_sql 'SELECT note FROM e2e_marker') == kept ]] || fail "the marker row is gone" +[[ $(https /e2e-custom) == 'custom route kept' ]] || fail "main's site does not answer again" +ok "main's layout starts again on its data" + +# --- The migration -------------------------------------------------------------------- + +step "The served launcher migrates it" +served self-update +expect_status 0 +expect_output "Moved from the released main layout to $RELEASE" +expect_output 'with a certificate from Caddy Local Authority' +setting() { "$S/ghost-docker" --dir "$S" config get "$1" 2>/dev/null; } +[[ $(setting COMPOSE_PROFILES) == production,activitypub ]] || fail "COMPOSE_PROFILES is $(setting COMPOSE_PROFILES)" +[[ $(setting COMPOSE_PROJECT_NAME) == "$project" ]] || fail "the project is $(setting COMPOSE_PROJECT_NAME)" +# shellcheck disable=SC2016 # a literal $, as Compose read it from main's .env +[[ $(setting DATABASE_PASSWORD) == 'e2e-app-pa$word' ]] || fail "the database password changed" +[[ $(setting URL) == "https://$DOMAIN" ]] || fail "URL is $(setting URL)" +[[ $("$S/ghost-docker" --dir "$S" config get ghost.env mail__options__auth__pass 2>/dev/null) == e2e-smtp-secret ]] || + fail "the SMTP password did not move to ghost.env" +grep -q '^DOMAIN=' "$S/.env" && fail ".env still has DOMAIN" +grep -q 'local_certs' "$S/caddy/global/legacy.caddy" || fail "the global options were not carried" +grep -q "activitypub-$project:8080" "$S/caddy/sites/site.caddy" || fail "ActivityPub is not this site's" "$(cat "$S/caddy/sites/site.caddy")" +cmp -s "$S/caddy/Caddyfile" "$ROOT/caddy/Caddyfile" || fail "caddy/Caddyfile is not the release's" +[[ -f $S/caddy/Caddyfile.local ]] || fail "the old Caddyfile was not kept" +running=$(docker inspect -f '{{.Config.Image}}' "$(ghost_id)") +[[ $running == "$(setting GHOST_IMAGE_REF)" && $running == ghost@sha256:* ]] || fail "Ghost runs $running" +now=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$(ghost_id)") +grep -qx "GHOST_VERSION=$version" <<<"$now" || fail "Ghost's version changed" "$now" +grep -qx 'e2e__override=kept' <<<"$now" || fail "the override does not reach Ghost" "$now" +grep -q 'DATABASE_ROOT_PASSWORD' <<<"$now" && fail "Ghost receives .env's root password" +[[ $(root_sql 'SELECT note FROM e2e_marker') == kept ]] || fail "the marker row is gone" +[[ $(compose_in exec -T ghost cat /home/ghost/content/images/e2e-marker.txt) == kept ]] || fail "the content is gone" +[[ $(https /e2e-custom) == 'custom route kept' ]] || fail "the custom route does not answer" "$(https /e2e-custom)" +[[ $(https /ghost/api/admin/site/) == *"\"url\":\"https://$DOMAIN/\""* ]] || fail "Ghost does not answer through Caddy" +docker volume inspect "${project}_caddy_data" >/dev/null || fail "Caddy's volume is not the site's" +[[ $(find "$S/backups" -mindepth 1 -maxdepth 1 -type d | wc -l) -eq 1 ]] || fail "there is not one backup" +[[ ! -e $S/.ghost-docker-update && ! -e $S/.ghost-docker.lock ]] || fail "the migration left its snapshot or lock" +ok "Ghost $version on its next image, project, volumes, data, routes and override kept" + +run "$S/ghost-docker" --dir "$S" check +expect_status 0 +ok "check passes" + +step "Running it again is an ordinary self-update, with nothing to do" +before=$(fingerprint) +served self-update +expect_status 0 +expect_output "already runs $RELEASE" +[[ $(fingerprint) == "$before" ]] || fail "a second run changed files" +ok "nothing to do" + +passed "migrate-main.sh: all checks passed." diff --git a/tests/fixtures/released-main/.env.example b/tests/fixtures/released-main/.env.example new file mode 100644 index 00000000..9056225c --- /dev/null +++ b/tests/fixtures/released-main/.env.example @@ -0,0 +1,70 @@ +# Use the below flags to enable the Analytics or ActivityPub containers as well +# COMPOSE_PROFILES=analytics,activitypub + +# Ghost domain +# Custom public domain Ghost will run on +DOMAIN=example.com + +# Ghost Admin domain +# If you have Ghost Admin setup on a separate domain uncomment the line below and add the domain +# You also need to uncomment the corresponding block in your Caddyfile +# ADMIN_DOMAIN= + +# Ghost ports +# Ports where Ghost will listen for HTTP traffic. +# Change these if the default ports are in use, or if Ghost is behind a reverse proxy. +HTTP_PORT=80 +HTTPS_PORT=443 + +# Database settings +# All database settings must not be changed once the database is initialised +DATABASE_ROOT_PASSWORD=reallysecurerootpassword +# DATABASE_USER=optionalusername +DATABASE_PASSWORD=ghostpassword + +# ActivityPub +# If you'd prefer to self-host ActivityPub yourself uncomment the line below +# ACTIVITYPUB_TARGET=activitypub:8080 + +# Tinybird configuration +# If you want to run Analytics, paste the output from `docker compose run --rm tinybird-login get-tokens` below +# TINYBIRD_API_URL=https://api.tinybird.co +# TINYBIRD_TRACKER_TOKEN=p.eyJxxxxx +# TINYBIRD_ADMIN_TOKEN=p.eyJxxxxx +# TINYBIRD_WORKSPACE_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + +# Shared secret for Ghost to authenticate Tinybird sync requests to Traffic Analytics +# for automation analytics. +# TINYBIRD_SYNC_AUTH=xxxxxxxx + +# Ghost configuration (https://ghost.org/docs/config/) + +# SMTP Email (https://ghost.org/docs/config/#mail) +# Transactional email is required for logins, account creation (staff invites), password resets and other features +# This is not related to bulk mail / newsletter sending +mail__transport=SMTP +mail__options__host=smtp.example.com +mail__options__port=465 +mail__options__secure=true +mail__options__auth__user=support@example.com +mail__options__auth__pass=1234567890 +mail__from="'Acme Support' " + +# Advanced customizations + +# Force Ghost version +# You should only do this if you need to pin a specific version +# The update commands won't work +# GHOST_VERSION=6-alpine + +# Port Ghost should listen on +# You should only need to edit this if you want to host +# multiple sites on the same server +# GHOST_PORT=2368 + +# Data locations +# Location to store uploaded data +UPLOAD_LOCATION=./data/ghost + +# Location for database data +MYSQL_DATA_LOCATION=./data/mysql diff --git a/tests/fixtures/released-main/README.md b/tests/fixtures/released-main/README.md new file mode 100644 index 00000000..14cf77a9 --- /dev/null +++ b/tests/fixtures/released-main/README.md @@ -0,0 +1,13 @@ +# The released `main` layout + +The stack files of the layout released on `main`, as of commit `9871293`, for +the tests of migration `0001-compose-profiles` (`manager/src/legacy.ts`). An +installation of it is a git clone of these with an untracked `.env` and +`caddy/Caddyfile`, copied from the examples and edited. + +Refresh them from `main` if it changes before `next-docker` is merged into it: + +```bash +git archive origin/main compose.yml compose.ipv6.yml .env.example caddy mysql-init \ + | tar -x -C tests/fixtures/released-main +``` diff --git a/tests/fixtures/released-main/caddy/Caddyfile.example b/tests/fixtures/released-main/caddy/Caddyfile.example new file mode 100644 index 00000000..17b6e027 --- /dev/null +++ b/tests/fixtures/released-main/caddy/Caddyfile.example @@ -0,0 +1,60 @@ +{$DOMAIN} { + import snippets/Logging + + # Traffic Analytics service + import snippets/TrafficAnalytics + + # ActivityPub Service + import snippets/ActivityPub + + # Default proxy everything else to Ghost + handle { + reverse_proxy ghost:2368 + } + + # Optional: Enable gzip compression + encode gzip + + # Optional: Add security headers + import snippets/SecurityHeaders +} + +# Separate admin domains +# To use a separate domain for Ghost Admin uncomment the block below (recommended) +# {$ADMIN_DOMAIN} { +# import snippets/Logging +# +# # Traffic Analytics service +# import snippets/TrafficAnalytics +# +# # ActivityPub Service +# import snippets/ActivityPub +# +# # Default proxy everything else to Ghost +# handle { +# reverse_proxy ghost:2368 +# } +# +# # Optional: Enable gzip compression +# encode gzip +# +# # Optional: Add security headers +# import snippets/SecurityHeaders +# } + +# Redirect www -> root domain +# To redirect the www variant of your domain to the non-www variant uncomment the 4 lines below +# Note: You must have DNS setup correctly for both domains for this to work +# www.{$DOMAIN} { +# import snippets/Logging +# redir https://{$DOMAIN}{uri} +# } + +# Redirect root -> www domain +# To redirect the non-www variant of your domain to the www variant uncomment the 4 lines below and change CHANGE_ME to your root domain +# Note: You must have DNS setup correctly for both domains for this to work +# When using ActivityPub with a www. domain, you must enable this redirect for ActivityPub to work correctly +# CHANGE_ME { +# import snippets/Logging +# redir https://{$DOMAIN}{uri} +# } diff --git a/tests/fixtures/released-main/caddy/snippets/ActivityPub b/tests/fixtures/released-main/caddy/snippets/ActivityPub new file mode 100644 index 00000000..e62d651e --- /dev/null +++ b/tests/fixtures/released-main/caddy/snippets/ActivityPub @@ -0,0 +1,13 @@ +# ActivityPub +# Proxy activitypub requests /.ghost/activitypub/ +handle /.ghost/activitypub/* { + reverse_proxy {$ACTIVITYPUB_TARGET} +} + +handle /.well-known/webfinger { + reverse_proxy {$ACTIVITYPUB_TARGET} +} + +handle /.well-known/nodeinfo { + reverse_proxy {$ACTIVITYPUB_TARGET} +} diff --git a/tests/fixtures/released-main/caddy/snippets/Logging b/tests/fixtures/released-main/caddy/snippets/Logging new file mode 100644 index 00000000..a0d6a666 --- /dev/null +++ b/tests/fixtures/released-main/caddy/snippets/Logging @@ -0,0 +1,6 @@ +# Log all requests +log { + output stdout + format console + level INFO +} diff --git a/tests/fixtures/released-main/caddy/snippets/SecurityHeaders b/tests/fixtures/released-main/caddy/snippets/SecurityHeaders new file mode 100644 index 00000000..a0b0d4ef --- /dev/null +++ b/tests/fixtures/released-main/caddy/snippets/SecurityHeaders @@ -0,0 +1,12 @@ +header { + # Enable HSTS + Strict-Transport-Security max-age=31536000; + # Enable XSS protection + X-XSS-Protection "1; mode=block" + # Prevent MIME sniffing + X-Content-Type-Options nosniff + # Referrer policy + Referrer-Policy strict-origin-when-cross-origin + # Prevent embedding in external iframes + Content-Security-Policy "frame-ancestors 'self' {$ADMIN_DOMAIN:}" +} diff --git a/tests/fixtures/released-main/caddy/snippets/TrafficAnalytics b/tests/fixtures/released-main/caddy/snippets/TrafficAnalytics new file mode 100644 index 00000000..53a81f2c --- /dev/null +++ b/tests/fixtures/released-main/caddy/snippets/TrafficAnalytics @@ -0,0 +1,6 @@ +# Proxy analytics requests with any prefix (e.g. /.ghost/analytics/ or /blog/.ghost/analytics/) +@analytics_paths path_regexp analytics_match ^(.*)/\.ghost/analytics(.*)$ +handle @analytics_paths { + rewrite * {re.analytics_match.2} + reverse_proxy traffic-analytics:3000 +} diff --git a/tests/fixtures/released-main/compose.ipv6.yml b/tests/fixtures/released-main/compose.ipv6.yml new file mode 100644 index 00000000..e2385f83 --- /dev/null +++ b/tests/fixtures/released-main/compose.ipv6.yml @@ -0,0 +1,6 @@ +networks: + ghost_network: + enable_ipv6: true + ipam: + config: + - subnet: fdc1:f5e4:6668::/64 diff --git a/tests/fixtures/released-main/compose.yml b/tests/fixtures/released-main/compose.yml new file mode 100644 index 00000000..e56ea3c9 --- /dev/null +++ b/tests/fixtures/released-main/compose.yml @@ -0,0 +1,209 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/compose-spec/compose-spec/main/schema/compose-spec.json +services: + caddy: + image: caddy:2.10.2-alpine@sha256:953131cfea8e12bfe1c631a36308e9660e4389f0c3dfb3be957044d3ac92d446 + restart: always + ports: + - "${HTTP_PORT:-80}:80" + - "${HTTPS_PORT:-443}:443" + environment: + DOMAIN: ${DOMAIN:?DOMAIN environment variable is required} + ADMIN_DOMAIN: ${ADMIN_DOMAIN:-} + ACTIVITYPUB_TARGET: ${ACTIVITYPUB_TARGET:-https://ap.ghost.org} + volumes: + - ./caddy:/etc/caddy + - caddy_data:/data + - caddy_config:/config + depends_on: + - ghost + networks: + - ghost_network + + ghost: + # Do not alter this without updating the Tinybird Sync container as well + image: ghost:${GHOST_VERSION:-6-alpine} + restart: always + # This is required to import current config when migrating + env_file: + - .env + environment: + NODE_ENV: production + url: https://${DOMAIN:?DOMAIN environment variable is required} + admin__url: ${ADMIN_DOMAIN:+https://${ADMIN_DOMAIN}} + database__client: mysql + database__connection__host: db + database__connection__user: ${DATABASE_USER:-ghost} + database__connection__password: ${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required} + database__connection__database: ghost + tinybird__tracker__endpoint: https://${DOMAIN:?DOMAIN environment variable is required}/.ghost/analytics/api/v1/page_hit + tinybird__adminToken: ${TINYBIRD_ADMIN_TOKEN:-} + tinybird__workspaceId: ${TINYBIRD_WORKSPACE_ID:-} + tinybird__sync_auth_key: ${TINYBIRD_SYNC_AUTH:-} + tinybird__tracker__datasource: analytics_events + tinybird__stats__endpoint: ${TINYBIRD_API_URL:-https://api.tinybird.co} + volumes: + - ${UPLOAD_LOCATION:-./data/ghost}:/var/lib/ghost/content + depends_on: + db: + condition: service_healthy + tinybird-sync: + condition: service_completed_successfully + required: false + tinybird-deploy: + condition: service_completed_successfully + required: false + activitypub: + condition: service_started + required: false + networks: + - ghost_network + + db: + image: mysql:8.0.44@sha256:f37951fc3753a6a22d6c7bf6978c5e5fefcf6f31814d98c582524f98eae52b21 + restart: always + expose: + - "3306" + environment: + MYSQL_ROOT_PASSWORD: ${DATABASE_ROOT_PASSWORD:?DATABASE_ROOT_PASSWORD environment variable is required} + MYSQL_USER: ${DATABASE_USER:-ghost} + MYSQL_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required} + MYSQL_DATABASE: ghost + MYSQL_MULTIPLE_DATABASES: activitypub + volumes: + - ${MYSQL_DATA_LOCATION:-./data/mysql}:/var/lib/mysql + - ./mysql-init:/docker-entrypoint-initdb.d + healthcheck: + test: mysqladmin ping -p$$MYSQL_ROOT_PASSWORD -h 127.0.0.1 + interval: 1s + start_period: 30s + start_interval: 10s + retries: 120 + networks: + - ghost_network + + traffic-analytics: + image: ghost/traffic-analytics:1.0.461@sha256:6d9324fb1fe43037338c7bd5ef5ff04390b16dad449cfc48831ebb00ddae9652 + restart: always + expose: + - "3000" + volumes: + - traffic_analytics_data:/data + environment: + NODE_ENV: production + PROXY_TARGET: ${TINYBIRD_API_URL:-https://api.tinybird.co}/v0/events + SALT_STORE_TYPE: ${SALT_STORE_TYPE:-file} + SALT_STORE_FILE_PATH: /data/salts.json + TINYBIRD_TRACKER_TOKEN: ${TINYBIRD_TRACKER_TOKEN:-} + TINYBIRD_SYNC_AUTH: ${TINYBIRD_SYNC_AUTH:-} + LOG_LEVEL: debug + profiles: [analytics] + networks: + - ghost_network + + activitypub: + image: ghcr.io/tryghost/activitypub:1.2.14@sha256:a607c8cd82d7d84eabec7abdb5c98e2788605572a0fa79ba75a872787e0084bc + restart: always + expose: + - "8080" + volumes: + - ${UPLOAD_LOCATION:-./data/ghost}:/opt/activitypub/content + environment: + # See https://github.com/TryGhost/ActivityPub/blob/main/docs/env-vars.md + NODE_ENV: production + MYSQL_HOST: db + MYSQL_USER: ${DATABASE_USER:-ghost} + MYSQL_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required} + MYSQL_DATABASE: activitypub + LOCAL_STORAGE_PATH: /opt/activitypub/content/images/activitypub + LOCAL_STORAGE_HOSTING_URL: https://${DOMAIN}/content/images/activitypub + depends_on: + db: + condition: service_healthy + activitypub-migrate: + condition: service_completed_successfully + profiles: [activitypub] + networks: + - ghost_network + + # Supporting Services + + tinybird-login: + build: + context: ./tinybird + dockerfile: Dockerfile + working_dir: /home/tinybird + command: /usr/local/bin/tinybird-login + volumes: + - tinybird_home:/home/tinybird + - tinybird_files:/data/tinybird + profiles: [analytics] + networks: + - ghost_network + tty: false + restart: no + + tinybird-sync: + # Do not alter this without updating the Ghost container as well + image: ghost:${GHOST_VERSION:-6-alpine} + command: > + sh -c " + if [ -d /var/lib/ghost/current/core/server/data/tinybird ]; then + rm -rf /data/tinybird/*; + cp -rf /var/lib/ghost/current/core/server/data/tinybird/* /data/tinybird/; + echo 'Tinybird files synced into shared volume.'; + else + echo 'Tinybird source directory not found.'; + fi + " + volumes: + - tinybird_files:/data/tinybird + depends_on: + tinybird-login: + condition: service_completed_successfully + networks: + - ghost_network + profiles: [analytics] + restart: no + + tinybird-deploy: + build: + context: ./tinybird + dockerfile: Dockerfile + working_dir: /data/tinybird + command: > + sh -c " + tb-wrapper --cloud deploy + " + volumes: + - tinybird_home:/home/tinybird + - tinybird_files:/data/tinybird + depends_on: + tinybird-sync: + condition: service_completed_successfully + profiles: [analytics] + networks: + - ghost_network + tty: true + + activitypub-migrate: + image: ghcr.io/tryghost/activitypub-migrations:1.2.14@sha256:c116a15f9c8a94542440fadc17bc06c15eeee6e3a61ff2731e3b6e2ea91f7356 + environment: + MYSQL_DB: mysql://${DATABASE_USER:-ghost}:${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required}@tcp(db:3306)/activitypub + networks: + - ghost_network + depends_on: + db: + condition: service_healthy + profiles: [activitypub] + restart: no + +volumes: + caddy_data: + caddy_config: + tinybird_files: + tinybird_home: + traffic_analytics_data: + +networks: + ghost_network: diff --git a/tests/fixtures/released-main/mysql-init/create-multiple-databases.sh b/tests/fixtures/released-main/mysql-init/create-multiple-databases.sh new file mode 100755 index 00000000..6b0d32ea --- /dev/null +++ b/tests/fixtures/released-main/mysql-init/create-multiple-databases.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +set -u + +if [ -n "$MYSQL_MULTIPLE_DATABASES" ]; then + echo "Creating multiple databases: $MYSQL_MULTIPLE_DATABASES" + + for db in $(echo "$MYSQL_MULTIPLE_DATABASES" | tr ',' ' '); do + echo "Creating database: $db" + mysql -u root -p"$MYSQL_ROOT_PASSWORD" <<-EOSQL + CREATE DATABASE IF NOT EXISTS \`$db\`; + GRANT ALL ON \`$db\`.* TO '$MYSQL_USER'@'%'; +EOSQL + done + + echo "Multiple databases created" +fi From 558ddfdeae80bbf9d727571f9fae869efe1041e3 Mon Sep 17 00:00:00 2001 From: Austin Burdine Date: Fri, 9 Oct 2026 15:51:44 -0400 Subject: [PATCH 3/5] Carried main's Caddyfile as written instead of translating it ref https://linear.app/ghost/issue/PLA-482 Translating the operator's Caddyfile into this layout's shape (snippet arguments, aliased upstreams, brace tracking) was the most fiddly part of the migration, and only made the routes look like a fresh install's. Keeping them working needs less: fill in the three variables main's Caddy read from its environment, point `import snippets/` at main's own snippets kept beside the routes, and lift a leading global options block into caddy/global/. Bare service upstreams still resolve on the site's own network. Caddy's validation remains the safety net. - manager/src/legacy/caddy.ts: the carry, about a third of the size. - manager/src/legacy.ts: keeps main's snippets in caddy/sites/legacy-snippets/, staged for validation too. - manager/test/, tests/e2e/migrate-main.sh: expectations for the carried routes. - docs/: what a moved site's routes look like, and that moving to this layout's snippets is optional until a shared Caddy. --- docs/architecture.md | 5 +- docs/caddy.md | 10 ++ docs/install.md | 2 +- manager/src/legacy.ts | 76 ++++----- manager/src/legacy/caddy.ts | 266 +++++------------------------- manager/test/legacy.test.ts | 160 +++++------------- manager/test/migrate-main.test.ts | 12 +- tests/e2e/migrate-main.sh | 5 +- 8 files changed, 155 insertions(+), 381 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index f509f834..a3d89272 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -260,7 +260,10 @@ managers. `src/legacy.ts` is migration `0001-compose-profiles`: the one way an installation of the released `main` layout (a clone with no metadata) reaches this layout, entered through the served launcher's `self-update`. `src/legacy/caddy.ts` -translates the operator's Caddyfile and `src/legacy/config.ts` splits its `.env`. +carries the operator's Caddyfile as written, with main's environment variables +filled in and main's snippets kept beside it rather than translated into this +layout's shape; Caddy decides whether it loads. `src/legacy/config.ts` splits +its `.env`. It decides everything before changing anything: Compose resolves the staged configuration with the operator's overrides and Caddy loads the staged routes. It then follows self-update's recovery boundary, reusing its snapshot, writer diff --git a/docs/caddy.md b/docs/caddy.md index 4b8c53d4..a838e602 100644 --- a/docs/caddy.md +++ b/docs/caddy.md @@ -25,6 +25,16 @@ import /etc/caddy/sites/*.caddy import /etc/caddy/custom/*.caddy ``` +## Routes moved from the released main layout + +A site moved from `main` keeps its own Caddyfile as `caddy/sites/site.caddy` +([install.md](install.md#moving-from-the-released-main-layout)). It imports +main's snippets, which take no arguments, from `caddy/sites/legacy-snippets/`, +and proxies to bare service names (`ghost:2368`), which resolve on the site's +own network. Both keep working. Moving to this layout's snippets and aliases, +as `install` writes them, is optional, and is needed before the site joins a +shared Caddy. + ## Changing routes Edit the file, then reload Caddy explicitly: diff --git a/docs/install.md b/docs/install.md index 9f6882b0..13928a0c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -548,7 +548,7 @@ resolve, if any of it cannot be carried over: | Credentials, ports, data locations, Tinybird settings | Kept in `.env`, with `SITE_MODE`, `PROJECT_DIR` and the rest of this layout's settings added | | Ghost's configuration in `.env` (`mail__*`, `labs__*`, anything Ghost read) | Moved to `ghost.env`, with the values Ghost actually received. Keys the container sets itself (`url`, `database__*`) are listed and left out, as they were overridden on main too | | `ghost:6-alpine` | The `next` image of **exactly the running version**, pinned by digest (`GHOST_IMAGE_REF`). Ghost 6.61.0 is the first with one: upgrade an older Ghost on main first (`docker compose pull ghost && docker compose up -d`) | -| `caddy/Caddyfile` | Kept as `caddy/Caddyfile.local`, and carried into `caddy/sites/site.caddy`: `{$DOMAIN}`-style variables filled in, snippet imports given their absolute paths and arguments, bare upstreams (`ghost:2368`) pointed at the site's aliases. Everything else you wrote is kept. A global options block moves to `caddy/global/legacy.caddy`. Caddy loads the result before anything changes | +| `caddy/Caddyfile` | Kept as `caddy/Caddyfile.local`, and carried into `caddy/sites/site.caddy` as written: `{$DOMAIN}`, `{$ADMIN_DOMAIN}` and `{$ACTIVITYPUB_TARGET}` filled in, and `import snippets/...` pointed at main's snippets, kept in `caddy/sites/legacy-snippets/`. A leading global options block moves to `caddy/global/legacy.caddy`. Caddy loads the result before anything changes; if it does not load, nothing is changed | | `compose.override.yml`, `GD_COMPOSE_OVERRIDES` | Kept, and resolved with the new layout before anything changes | | Stack files (`compose.yml`, snippets, `mysql-init/`) | The release's. Git must show them unedited: a change in the work tree or in a commit no remote has stops the migration. Move such changes into `compose.override.yml` or a `.caddy` file of your own afterwards | diff --git a/manager/src/legacy.ts b/manager/src/legacy.ts index fb706cea..964724c2 100644 --- a/manager/src/legacy.ts +++ b/manager/src/legacy.ts @@ -32,6 +32,7 @@ import { cpSync, existsSync, mkdirSync, + readdirSync, readFileSync, rmSync, statSync, @@ -61,8 +62,9 @@ import { GLOBAL_FILE, KEPT_CADDYFILE, LEGACY_CADDYFILE, - translateCaddyfile, - UntranslatableCaddyfile, + carryCaddyfile, + fillEnvironment, + LEGACY_SNIPPETS, } from './legacy/caddy.ts'; import { readLegacyEnv, @@ -150,7 +152,8 @@ interface Plan { readonly split: SplitEnv; readonly routes: string; readonly global: string | null; - readonly caddyChanges: readonly string[]; + /** main's snippets, by name, for LEGACY_SNIPPETS. */ + readonly snippets: Readonly>; readonly hadCaddyfile: boolean; /** Overrides other than compose.override.yml, relative to the site. */ readonly overrides: string[]; @@ -418,55 +421,42 @@ async function prepare( } ok(io, 'configuration', `.env and ghost.env for a production site, resolved by Compose`); - // The routes, loaded by the Caddy image the site will run. - // Where main sent ActivityPub, which with the profile on was still the - // hosted service unless ACTIVITYPUB_TARGET said otherwise: moving it would - // move the site's followers. - const activitypub = (old.ACTIVITYPUB_TARGET || HOSTED_ACTIVITYPUB).replace( - /^(?:https?:\/\/)?activitypub:8080$/, - `activitypub-${project}:8080`, - ); + // The routes, loaded by the Caddy image the site will run. ActivityPub + // goes where main sent it, which with the profile on was still the hosted + // service unless ACTIVITYPUB_TARGET said otherwise: moving it would move + // the site's followers. + const values = { + domain, + adminDomain, + activitypub: old.ACTIVITYPUB_TARGET || HOSTED_ACTIVITYPUB, + }; const original = readIfExists(join(dir, LEGACY_CADDYFILE)); let routes: string; let global: string | null = null; - let caddyChanges: readonly string[] = []; + // main's snippets, read before the release replaces them; git has shown them unedited. + const snippets: Record = {}; if (original === undefined) { routes = renderRoutes({ project, domain, adminDomain, email: '', - activitypub: activitypub === `activitypub-${project}:8080`, + activitypub: /^(?:https?:\/\/)?activitypub:8080$/.test(values.activitypub), }); } else { - try { - ({ - site: routes, - global, - changes: caddyChanges, - } = translateCaddyfile(original, { - project, - domain, - adminDomain, - activitypub, - })); - } catch (error) { - if (error instanceof UntranslatableCaddyfile) { - refuse( - `${error.message}\n Change caddy/Caddyfile so it can be carried over, or move what it does into\n` + - ' compose.override.yml or caddy/custom/ afterwards (docs/caddy.md).', - ); - } - throw error; + ({ site: routes, global } = carryCaddyfile(original, values)); + const kept = join(dir, 'caddy', 'snippets'); + for (const name of existsSync(kept) ? readdirSync(kept) : []) { + snippets[name] = fillEnvironment(readFileSync(join(kept, name), 'utf8'), values); } } - await validateRoutes(io, staged, stack, after, routes, global); + await validateRoutes(io, staged, stack, after, routes, global, snippets); ok( io, SITE_FILE, original === undefined ? 'written: there was no caddy/Caddyfile' - : `carried over from caddy/Caddyfile, and loaded by Caddy`, + : 'carried over from caddy/Caddyfile, and loaded by Caddy', ); return { @@ -487,7 +477,7 @@ async function prepare( split, routes, global, - caddyChanges, + snippets, hadCaddyfile: original !== undefined, overrides, port, @@ -575,10 +565,12 @@ async function validateRoutes( after: ResolvedSite, routes: string, global: string | null, + snippets: Readonly>, ): Promise { const caddy = join(staged, 'caddy'); cpSync(join(stack, 'caddy'), caddy, { recursive: true }); atomicWrite(join(staged, SITE_FILE), routes, 0o644); + writeSnippets(staged, snippets); if (global !== null) { atomicWrite(join(staged, GLOBAL_FILE), global, 0o644); } @@ -612,11 +604,21 @@ async function validateRoutes( refuse( 'Caddy does not load the routes carried over from caddy/Caddyfile:\n' + said.map((line) => ` ${line}\n`).join('') + - ' Change caddy/Caddyfile so they load, then run this again.', + ' They are carried as written, with DOMAIN, ADMIN_DOMAIN and ACTIVITYPUB_TARGET filled in\n' + + " and main's snippets kept beside them. Change caddy/Caddyfile so they load, then run\n" + + ' this again.', ); } } +/** main's snippets beside the carried routes, which import them. */ +function writeSnippets(root: string, snippets: Readonly>): void { + for (const [name, content] of Object.entries(snippets)) { + mkdirSync(join(root, LEGACY_SNIPPETS), { recursive: true, mode: 0o755 }); + atomicWrite(join(root, LEGACY_SNIPPETS, name), content, 0o644); + } +} + // --- --check ----------------------------------------------------------------------- function report(plan: Plan): number { @@ -633,7 +635,6 @@ function report(plan: Plan): number { ({ key, reason }) => ` ${key} not carried: ${reason}`, ), ` Routes ${SITE_FILE}${plan.hadCaddyfile ? ', from caddy/Caddyfile, kept as caddy/Caddyfile.local' : ''}`, - ...plan.caddyChanges.map((change) => ` ${change}`), ` Loopback Ghost published on 127.0.0.1:${plan.port}`, ` Launcher ./${LAUNCHER}, pinned to ${plan.pin}`, '', @@ -745,6 +746,7 @@ function write(plan: Plan): Metadata { printChecks(io, [{ status: 'note', label: 'not carried', detail: `${key}: ${reason}` }]); } atomicWrite(join(dir, SITE_FILE), plan.routes, 0o644); + writeSnippets(dir, plan.snippets); if (plan.global !== null) { atomicWrite(join(dir, GLOBAL_FILE), plan.global, 0o644); } diff --git a/manager/src/legacy/caddy.ts b/manager/src/legacy/caddy.ts index 85ae4130..7d19aaeb 100644 --- a/manager/src/legacy/caddy.ts +++ b/manager/src/legacy/caddy.ts @@ -1,90 +1,51 @@ // The operator's own Caddyfile from the released `main` layout, carried into -// this layout's routes (docs/install.md#moving-from-the-released-main-layout). +// this layout's routes as it is (docs/install.md#moving-from-the-released-main-layout). // -// On `main`, caddy/Caddyfile was copied from Caddyfile.example and then edited -// by hand. It read DOMAIN, ADMIN_DOMAIN and ACTIVITYPUB_TARGET from Caddy's -// environment, imported snippets by relative path with no arguments, and -// proxied to bare service names. This layout gives Caddy no environment, its -// snippets take arguments, and upstreams are the site's unique aliases. The -// translation changes exactly those things and keeps everything else the -// operator wrote, so their routes keep working; Caddy itself then validates -// the result before anything live is changed. +// On `main`, caddy/Caddyfile was copied from Caddyfile.example and edited by +// hand. It read DOMAIN, ADMIN_DOMAIN and ACTIVITYPUB_TARGET from Caddy's +// environment and imported main's snippets, which take no arguments, by +// relative path. Here Caddy has no environment and the snippets take +// arguments. Rather than rewrite the operator's routes into this layout's +// shape, the file is kept as written: the three variables are filled in, and +// its snippet imports point at main's snippets, kept beside it. Bare service +// upstreams (`ghost:2368`) still resolve on the site's own network. Caddy +// itself then decides whether the result loads, before anything is changed. import { join } from 'node:path'; /** The original, kept beside the routes it became; .gitignore leaves it out. */ export const LEGACY_CADDYFILE = join('caddy', 'Caddyfile'); export const KEPT_CADDYFILE = join('caddy', 'Caddyfile.local'); +/** main's snippets, which the carried routes import. Not `*.caddy`, so never loaded as sites. */ +export const LEGACY_SNIPPETS = join('caddy', 'sites', 'legacy-snippets'); /** A global options block, which this layout's Caddyfile imports from caddy/global/. */ export const GLOBAL_FILE = join('caddy', 'global', 'legacy.caddy'); -/** Services whose bare `name:port` is rewritten to the site's unique alias. */ -const ALIASED = ['ghost', 'db', 'traffic-analytics', 'activitypub'] as const; - -/** What the old Caddy container's environment held, as Compose set it on `main`. */ +/** What the old Caddy container's environment held, as main's compose.yml set it. */ export interface LegacyCaddyValues { - readonly project: string; readonly domain: string; readonly adminDomain: string; - /** ACTIVITYPUB_TARGET, already pointing at the site's alias when it was `activitypub:8080`. */ readonly activitypub: string; } -export interface TranslatedCaddyfile { +export interface CarriedCaddyfile { /** caddy/sites/site.caddy. */ readonly site: string; - /** The global options block's body, for GLOBAL_FILE; null when there was none. */ + /** A leading global options block's body, for GLOBAL_FILE; null when there was none. */ readonly global: string | null; - /** What was changed, for the operator. */ - readonly changes: readonly string[]; -} - -/** A Caddyfile this cannot carry over automatically; nothing has been changed. */ -export class UntranslatableCaddyfile extends Error {} - -/** Snippets the stack ships, and the arguments each now takes, in order. */ -function snippetArguments(name: string, values: LegacyCaddyValues): string | null { - switch (name) { - case 'Logging': - return ''; - case 'TrafficAnalytics': - return ` traffic-analytics-${values.project}:3000`; - case 'ActivityPub': - return ` ${values.activitypub}`; - case 'SecurityHeaders': - return ` "${values.adminDomain}"`; - default: - return null; - } -} - -/** Splits a line into its code and its comment: `#` starts one only at a token's start. */ -function splitComment(line: string): { code: string; comment: string } { - let quoted = false; - for (let i = 0; i < line.length; i += 1) { - const char = line[i]!; - if (char === '\\') { - i += 1; - } else if (char === '"') { - quoted = !quoted; - } else if (char === '#' && !quoted && (i === 0 || /\s/.test(line[i - 1]!))) { - return { code: line.slice(0, i), comment: line.slice(i) }; - } - } - return { code: line, comment: '' }; } /** - * `{$NAME}` and `{$NAME:default}` are substituted when the file is loaded, - * `{env.NAME}` when a request is served. Both read Caddy's environment, which - * no longer holds these three. + * `{$NAME}` and `{$NAME:default}`, filled in when the file is loaded, and + * `{env.NAME}`, when a request is served: both read Caddy's environment, + * which no longer holds these three. Anything else is left as written. */ -function substituteEnvironment(code: string, values: LegacyCaddyValues): string { +export function fillEnvironment(text: string, values: LegacyCaddyValues): string { const known: Record = { DOMAIN: values.domain, ADMIN_DOMAIN: values.adminDomain, ACTIVITYPUB_TARGET: values.activitypub, }; - return code + return text .replace(/\{\$([A-Za-z_][A-Za-z0-9_]*)(?::([^}]*))?\}/g, (whole, name: string, fallback) => name in known ? known[name] || (fallback ?? '') : whole, ) @@ -93,177 +54,38 @@ function substituteEnvironment(code: string, values: LegacyCaddyValues): string ); } -const ALIAS = new RegExp( - `(^|[\\s/])(${ALIASED.map((name) => name.replace('-', '\\-')).join('|')}):(\\d+)(?=$|[\\s/])`, - 'g', -); - /** - * An import of a shipped snippet gains its absolute path and arguments; any - * other relative import is made absolute, since it now sits in caddy/sites/ - * rather than beside the Caddyfile it was relative to. Named snippets the - * file defines are left alone. + * The operator's Caddyfile as caddy/sites/site.caddy. A global options block + * must come first in Caddy's whole configuration, which the stack's + * Caddyfile opens with its own, so one written first, `{` and `}` alone on + * their lines, moves to caddy/global/. Whatever else does not load, Caddy says. */ -function rewriteImport( - code: string, - values: LegacyCaddyValues, - named: ReadonlySet, - changes: Set, -): string { - const match = /^(\s*)import(\s+)(\S+)(.*)$/.exec(code); - if (match === null) { - return code; - } - const [, indent, space, path, rest] = match as unknown as [ - string, - string, - string, - string, - string, - ]; - if (named.has(path)) { - return code; - } - const snippet = /^(?:\.\/|\/etc\/caddy\/)?snippets\/([A-Za-z]+)$/.exec(path); - if (snippet !== null) { - const name = snippet[1]!; - const args = snippetArguments(name, values); - if (args !== null) { - if (rest.trim() !== '') { - throw new UntranslatableCaddyfile( - `caddy/Caddyfile imports snippets/${name} with arguments (${rest.trim()}); on main it took none,\n` + - ' so it cannot be told what they were meant to be.', - ); - } - changes.add('snippet imports given their absolute paths and arguments'); - return `${indent}import${space}/etc/caddy/snippets/${name}${args}`; - } - } - if (path.startsWith('/')) { - return code; - } - changes.add('relative imports made absolute'); - return `${indent}import${space}/etc/caddy/${path.replace(/^\.\//, '')}${rest}`; -} - -/** Does this line's code open a block, close one, or neither? */ -function braces(code: string): { opens: boolean; closes: boolean } { - const tokens = code.trim().split(/\s+/); - return { - opens: tokens.at(-1) === '{', - closes: tokens[0] === '}', - }; -} - -/** - * The operator's Caddyfile, as routes for this layout. Throws - * UntranslatableCaddyfile for what it cannot carry over; Caddy validates the - * rest before anything is written. - */ -export function translateCaddyfile(text: string, values: LegacyCaddyValues): TranslatedCaddyfile { - if (/<<[A-Za-z]/.test(text)) { - throw new UntranslatableCaddyfile( - 'caddy/Caddyfile uses a heredoc, which this migration does not translate', - ); - } - const lines = text.replace(/\r\n/g, '\n').split('\n'); - const named = new Set( - lines - .map((line) => /^\s*\(([^)\s]+)\)\s*\{\s*$/.exec(splitComment(line).code)?.[1]) - .filter((name): name is string => name !== undefined), +export function carryCaddyfile(text: string, values: LegacyCaddyValues): CarriedCaddyfile { + let routes = fillEnvironment(text.replace(/\r\n/g, '\n'), values).replace( + /^([ \t]*import[ \t]+)(?:\.\/)?snippets\//gm, + `$1/etc/caddy/sites/legacy-snippets/`, ); - const changes = new Set(); - - const site: string[] = []; - const global: string[] = []; - let depth = 0; - /** In the global options block, which is the first block and has no address. */ - let inGlobal = false; - let seenBlock = false; - for (const line of lines) { - const { code: original, comment } = splitComment(line); - let code = substituteEnvironment(original, values); - if (code !== original) { - changes.add( - '{$DOMAIN}, {$ADMIN_DOMAIN} and {$ACTIVITYPUB_TARGET} replaced by their values', - ); - } - const aliased = code.replace( - ALIAS, - (_, before: string, service: string, port: string) => - `${before}${service}-${values.project}:${port}`, - ); - if (aliased !== code) { - changes.add('bare service upstreams pointed at the site’s unique aliases'); - code = aliased; - } - code = rewriteImport(code, values, named, changes); - const { opens, closes } = braces(code); - - if (depth === 0 && opens) { - // Only a block written with no address is global options: one - // whose address is a variable that turned out empty is a mistake. - if (original.trim() === '{' && !seenBlock) { - inGlobal = true; - seenBlock = true; - depth += 1; - changes.add(`the global options block moved to ${GLOBAL_FILE}`); - continue; - } - const addresses = code - .trim() - .slice(0, -1) - .trim() - .split(/\s*,\s*|\s+/); - if (addresses.some((address) => address === '')) { - throw new UntranslatableCaddyfile( - `caddy/Caddyfile has a site block with an empty address once DOMAIN and ADMIN_DOMAIN are\n` + - ` filled in (${original.trim()}). Set ADMIN_DOMAIN in .env, or remove that block.`, - ); - } - } - if (closes) { - depth -= 1; - if (depth < 0) { - throw new UntranslatableCaddyfile( - 'caddy/Caddyfile closes more blocks than it opens', - ); - } - if (depth === 0 && inGlobal) { - inGlobal = false; - continue; - } - } - if (opens) { - depth += 1; - seenBlock = true; - } - if (inGlobal) { - // One level shallower: the stack's Caddyfile supplies the braces. - global.push(`${code}${comment}`.replace(/^\t/, '')); - } else { - site.push(`${code}${comment}`); + let global: string | null = null; + const lines = routes.split('\n'); + const first = lines.findIndex((line) => line.trim() !== '' && !line.trim().startsWith('#')); + if (first >= 0 && lines[first]!.trim() === '{') { + const close = lines.findIndex((line, index) => index > first && line === '}'); + if (close > first) { + global = `# Global options carried over from caddy/Caddyfile.\n${lines + .slice(first + 1, close) + .map((line) => line.replace(/^\t/, '')) + .join('\n') + .trim()}\n`; + routes = [...lines.slice(0, first), ...lines.slice(close + 1)].join('\n'); } } - if (depth !== 0) { - throw new UntranslatableCaddyfile('caddy/Caddyfile opens more blocks than it closes'); - } - const header = [ - `# Routes for ${values.project}, carried over from caddy/Caddyfile, which is kept as`, - '# caddy/Caddyfile.local. This file is yours: edit it, then reload Caddy:', + '# Routes carried over from caddy/Caddyfile, which is kept as caddy/Caddyfile.local.', + '# They import the released main layout’s snippets, kept in legacy-snippets/ beside', + '# this file. This file is yours: edit it, then reload Caddy:', '#', '# docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile', - '#', - '# docs/caddy.md lists the snippets and the arguments each one takes.', '', ]; - return { - site: `${[...header, ...site].join('\n').trimEnd()}\n`, - global: - global.length === 0 - ? null - : `# Global options carried over from caddy/Caddyfile.\n${global.join('\n').trimEnd()}\n`, - changes: [...changes], - }; + return { site: `${[...header, routes.trim()].join('\n')}\n`, global }; } diff --git a/manager/test/legacy.test.ts b/manager/test/legacy.test.ts index 415904d2..316114d8 100644 --- a/manager/test/legacy.test.ts +++ b/manager/test/legacy.test.ts @@ -5,11 +5,7 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { describe, test } from 'node:test'; -import { - translateCaddyfile, - UntranslatableCaddyfile, - type LegacyCaddyValues, -} from '../src/legacy/caddy.ts'; +import { carryCaddyfile, fillEnvironment, type LegacyCaddyValues } from '../src/legacy/caddy.ts'; import { readLegacyEnv, refuseInterpolated, splitLegacyEnv } from '../src/legacy/config.ts'; import { REPO } from './site.ts'; @@ -17,13 +13,12 @@ const MAIN = join(REPO, 'tests', 'fixtures', 'released-main'); const EXAMPLE = readFileSync(join(MAIN, 'caddy', 'Caddyfile.example'), 'utf8'); const VALUES: LegacyCaddyValues = { - project: 'ghost-docker', domain: 'example.com', adminDomain: '', activitypub: 'https://ap.ghost.org', }; -const translate = (text: string, values: Partial = {}) => - translateCaddyfile(text, { ...VALUES, ...values }); +const carry = (text: string, values: Partial = {}) => + carryCaddyfile(text, { ...VALUES, ...values }); /** The lines that are not comments: what Caddy loads. */ const code = (text: string) => @@ -32,91 +27,60 @@ const code = (text: string) => .filter((line) => line.trim() !== '' && !line.trim().startsWith('#')) .join('\n'); -/** Uncomment the example's block that starts with `first`. */ -function uncomment(text: string, first: string): string { - const lines = text.split('\n'); - let inside = false; - return lines - .map((line) => { - if (line.startsWith(`# ${first}`)) { - inside = true; - } - if (inside && /^#( |$)/.test(line)) { - const uncommented = line.replace(/^# ?/, ''); - if (uncommented === '}') { - inside = false; - } - return uncommented; - } - return line; - }) - .join('\n'); -} - describe('the Caddyfile', () => { - test('the example, as most sites have it, becomes routes with arguments and aliases', () => { - const { site, global, changes } = translate(EXAMPLE); + test('the example is carried as written, with its variables filled in and its snippets kept', () => { + const { site, global } = carry(EXAMPLE); const loaded = code(site); assert.match(loaded, /^example\.com \{$/m); - assert.match(loaded, /^\timport \/etc\/caddy\/snippets\/Logging$/m); - assert.match( - loaded, - /^\timport \/etc\/caddy\/snippets\/TrafficAnalytics traffic-analytics-ghost-docker:3000$/m, - ); - assert.match( - loaded, - /^\timport \/etc\/caddy\/snippets\/ActivityPub https:\/\/ap\.ghost\.org$/m, - ); - assert.match(loaded, /^\timport \/etc\/caddy\/snippets\/SecurityHeaders ""$/m); - assert.match(loaded, /^\t\treverse_proxy ghost-ghost-docker:2368$/m); - assert.doesNotMatch(loaded, /\{\$|import snippets\/|ghost:2368/); + for (const name of ['Logging', 'TrafficAnalytics', 'ActivityPub', 'SecurityHeaders']) { + assert.match( + loaded, + new RegExp(`^\\timport /etc/caddy/sites/legacy-snippets/${name}$`, 'm'), + ); + } + // Bare upstreams resolve on the site's own network. + assert.match(loaded, /^\t\treverse_proxy ghost:2368$/m); + assert.doesNotMatch(loaded, /\{\$|import snippets\//); assert.equal(global, null); - assert.ok(changes.length > 0); - }); - - test("the admin domain's block is filled in, and may frame the site", () => { - const { site } = translate(uncomment(EXAMPLE, '{$ADMIN_DOMAIN}'), { - adminDomain: 'admin.example.com', - }); - assert.match(code(site), /^admin\.example\.com \{$/m); - assert.equal(code(site).match(/SecurityHeaders "admin\.example\.com"/g)?.length, 2); - }); - - test('an admin block with no ADMIN_DOMAIN is refused, not turned into global options', () => { - assert.throws( - () => translate(uncomment(EXAMPLE, '{$ADMIN_DOMAIN}')), - (error) => - error instanceof UntranslatableCaddyfile && /empty address/.test(error.message), + // Everything else, comments included, as it was. + assert.equal( + code(site), + code( + EXAMPLE.replaceAll('{$DOMAIN}', 'example.com').replace( + /import snippets\//g, + 'import /etc/caddy/sites/legacy-snippets/', + ), + ), ); }); - test('the www redirect keeps redirecting to the domain', () => { - const { site } = translate(uncomment(EXAMPLE, 'www.{$DOMAIN}')); - assert.match(code(site), /^www\.example\.com \{$/m); - assert.match(code(site), /^\tredir https:\/\/example\.com\{uri\}$/m); - }); - - test("the site's own ActivityPub is addressed by its alias", () => { - const { site } = translate(EXAMPLE, { activitypub: 'activitypub-ghost-docker:8080' }); - assert.match(code(site), /snippets\/ActivityPub activitypub-ghost-docker:8080$/m); - // Written straight into a route, the bare name is rewritten too. - const direct = translate('example.com {\n\treverse_proxy http://activitypub:8080\n}\n'); - assert.match(direct.site, /reverse_proxy http:\/\/activitypub-ghost-docker:8080/); + test("main's snippets get the same variables", () => { + const headers = readFileSync(join(MAIN, 'caddy', 'snippets', 'SecurityHeaders'), 'utf8'); + assert.match( + fillEnvironment(headers, { ...VALUES, adminDomain: 'admin.example.com' }), + /frame-ancestors 'self' admin\.example\.com/, + ); + assert.match(fillEnvironment(headers, VALUES), /frame-ancestors 'self' "/); + const activitypub = readFileSync(join(MAIN, 'caddy', 'snippets', 'ActivityPub'), 'utf8'); + assert.match( + fillEnvironment(activitypub, { ...VALUES, activitypub: 'activitypub:8080' }), + /reverse_proxy activitypub:8080/, + ); }); test('custom routes are kept as written', () => { - const custom = EXAMPLE.replace( - '\t# Default proxy everything else to Ghost', - '\tredir /old /new 301\n\thandle /status {\n\t\trespond "ok"\n\t}\n\n\t# Default proxy everything else to Ghost', - ).concat('\nstatus.example.com {\n\treverse_proxy 172.17.0.1:9000\n}\n'); - const { site } = translate(custom); - assert.match(site, /^\tredir \/old \/new 301$/m); - assert.match(site, /^\thandle \/status \{\n\t\trespond "ok"\n\t\}$/m); - assert.match(site, /^status\.example\.com \{\n\treverse_proxy 172\.17\.0\.1:9000\n\}$/m); + const custom = `${EXAMPLE}\nstatus.example.com {\n\theader X-Host {env.DOMAIN}\n\theader X-Other {$OTHER}\n\treverse_proxy 172.17.0.1:9000\n}\n`; + const { site } = carry(custom); + assert.match( + site, + /^status\.example\.com \{\n\theader X-Host example\.com\n\theader X-Other \{\$OTHER\}\n\treverse_proxy 172\.17\.0\.1:9000\n\}$/m, + ); }); - test('a global options block moves to caddy/global, one level shallower', () => { - const { site, global } = translate(`{\n\temail ops@example.com\n\tdebug\n}\n\n${EXAMPLE}`); + test('a leading global options block moves to caddy/global, one level shallower', () => { + const { site, global } = carry( + `# mine\n{\n\temail ops@example.com\n\tdebug\n}\n\n${EXAMPLE}`, + ); assert.equal( global, '# Global options carried over from caddy/Caddyfile.\nemail ops@example.com\ndebug\n', @@ -124,40 +88,6 @@ describe('the Caddyfile', () => { assert.doesNotMatch(code(site), /email ops/); assert.match(code(site), /^example\.com \{$/m); }); - - test('named snippets stay named; other relative imports become absolute', () => { - const { site } = translate( - '(common) {\n\tencode gzip\n}\n\nexample.com {\n\timport common\n\timport extra/*.caddy\n\treverse_proxy ghost:2368\n}\n', - ); - assert.match(site, /^\timport common$/m); - assert.match(site, /^\timport \/etc\/caddy\/extra\/\*\.caddy$/m); - }); - - test('runtime placeholders and other variables are left alone', () => { - const { site } = translate( - 'example.com {\n\theader X-Host {env.DOMAIN}\n\theader X-Other {$OTHER} {uri}\n}\n', - ); - assert.match(site, /X-Host example\.com$/m); - assert.match(site, /X-Other \{\$OTHER\} \{uri\}$/m); - }); - - test('a snippet imported with arguments it never took is refused', () => { - assert.throws( - () => translate('example.com {\n\timport snippets/Logging verbose\n}\n'), - UntranslatableCaddyfile, - ); - }); - - test('unbalanced braces are refused', () => { - assert.throws(() => translate('example.com {\n\trespond ok\n'), UntranslatableCaddyfile); - assert.throws(() => translate('}\n'), UntranslatableCaddyfile); - }); - - test('comments are kept as they were', () => { - const { site } = translate(EXAMPLE); - assert.match(site, /^# \{\$ADMIN_DOMAIN\} \{$/m); - assert.match(site, /^\t# Optional: Enable gzip compression$/m); - }); }); describe('the .env', () => { diff --git a/manager/test/migrate-main.test.ts b/manager/test/migrate-main.test.ts index 0c891748..8c700aaf 100644 --- a/manager/test/migrate-main.test.ts +++ b/manager/test/migrate-main.test.ts @@ -230,8 +230,12 @@ describe('migrating the released main layout', () => { const routes = readSite('caddy/sites/site.caddy'); assert.match(routes, /^example\.com \{$/m); - assert.match(routes, new RegExp(`snippets/ActivityPub activitypub-${project}:8080$`, 'm')); - assert.match(routes, new RegExp(`reverse_proxy ghost-${project}:2368$`, 'm')); + assert.match(routes, /^\timport \/etc\/caddy\/sites\/legacy-snippets\/ActivityPub$/m); + assert.match(routes, /reverse_proxy ghost:2368$/m); + assert.match( + readSite('caddy/sites/legacy-snippets/ActivityPub'), + /reverse_proxy activitypub:8080$/m, + ); assert.equal( readSite('caddy/Caddyfile.local'), readFileSync(join(MAIN, 'caddy', 'Caddyfile.example'), 'utf8'), @@ -266,8 +270,8 @@ describe('migrating the released main layout', () => { const result = await migrate(); assert.equal(result.code, 0, result.stderr); assert.match( - readSite('caddy/sites/site.caddy'), - /snippets\/ActivityPub https:\/\/ap\.ghost\.org$/m, + readSite('caddy/sites/legacy-snippets/ActivityPub'), + /reverse_proxy https:\/\/ap\.ghost\.org$/m, ); assert.equal(envOf('.env').COMPOSE_PROFILES, 'production,activitypub'); }); diff --git a/tests/e2e/migrate-main.sh b/tests/e2e/migrate-main.sh index b0661d73..194b709f 100755 --- a/tests/e2e/migrate-main.sh +++ b/tests/e2e/migrate-main.sh @@ -255,7 +255,10 @@ setting() { "$S/ghost-docker" --dir "$S" config get "$1" 2>/dev/null; } fail "the SMTP password did not move to ghost.env" grep -q '^DOMAIN=' "$S/.env" && fail ".env still has DOMAIN" grep -q 'local_certs' "$S/caddy/global/legacy.caddy" || fail "the global options were not carried" -grep -q "activitypub-$project:8080" "$S/caddy/sites/site.caddy" || fail "ActivityPub is not this site's" "$(cat "$S/caddy/sites/site.caddy")" +grep -q 'import /etc/caddy/sites/legacy-snippets/ActivityPub' "$S/caddy/sites/site.caddy" || + fail "the routes do not import main's snippets" "$(cat "$S/caddy/sites/site.caddy")" +grep -q 'reverse_proxy activitypub:8080' "$S/caddy/sites/legacy-snippets/ActivityPub" || + fail "ActivityPub is not where main sent it" "$(cat "$S/caddy/sites/legacy-snippets/ActivityPub")" cmp -s "$S/caddy/Caddyfile" "$ROOT/caddy/Caddyfile" || fail "caddy/Caddyfile is not the release's" [[ -f $S/caddy/Caddyfile.local ]] || fail "the old Caddyfile was not kept" running=$(docker inspect -f '{{.Config.Image}}' "$(ghost_id)") From 98349f6b523588c5653bc69a6dcefa12c5e2476e Mon Sep 17 00:00:00 2001 From: Austin Burdine Date: Fri, 9 Oct 2026 16:23:13 -0400 Subject: [PATCH 4/5] Read the released main layout from git instead of a fixture copy ref https://linear.app/ghost/issue/PLA-482 main keeps changing (Renovate bumps, the occasional fix) until next-docker is merged into it, and a copied fixture would drift from it unnoticed. The migration tests now read main itself: the unit tests check out GD_RELEASED_MAIN (origin/main) from this repository, and the e2e scenario clones the repository and checks it out, which is also exactly how an operator's installation came to be. S12 pins the ref to main's last commit before the merge. - manager/test/site.ts: releasedMain(), main's files from git. - manager/test/legacy.test.ts, migrate-main.test.ts: read them. - tests/e2e/migrate-main.sh: a real clone of main. - .github/workflows/test.yml: the manager job fetches main. - tests/fixtures/released-main/: removed. - docs/ghost-cli-replacement.md: S12 pins GD_RELEASED_MAIN. --- .github/workflows/test.yml | 3 + docs/ghost-cli-replacement.md | 5 +- manager/test/legacy.test.ts | 4 +- manager/test/migrate-main.test.ts | 3 +- manager/test/site.ts | 37 ++++ tests/e2e/migrate-main.sh | 24 +- tests/fixtures/released-main/.env.example | 70 ------ tests/fixtures/released-main/README.md | 13 -- .../released-main/caddy/Caddyfile.example | 60 ----- .../released-main/caddy/snippets/ActivityPub | 13 -- .../released-main/caddy/snippets/Logging | 6 - .../caddy/snippets/SecurityHeaders | 12 - .../caddy/snippets/TrafficAnalytics | 6 - tests/fixtures/released-main/compose.ipv6.yml | 6 - tests/fixtures/released-main/compose.yml | 209 ------------------ .../mysql-init/create-multiple-databases.sh | 18 -- 16 files changed, 58 insertions(+), 431 deletions(-) delete mode 100644 tests/fixtures/released-main/.env.example delete mode 100644 tests/fixtures/released-main/README.md delete mode 100644 tests/fixtures/released-main/caddy/Caddyfile.example delete mode 100644 tests/fixtures/released-main/caddy/snippets/ActivityPub delete mode 100644 tests/fixtures/released-main/caddy/snippets/Logging delete mode 100644 tests/fixtures/released-main/caddy/snippets/SecurityHeaders delete mode 100644 tests/fixtures/released-main/caddy/snippets/TrafficAnalytics delete mode 100644 tests/fixtures/released-main/compose.ipv6.yml delete mode 100644 tests/fixtures/released-main/compose.yml delete mode 100755 tests/fixtures/released-main/mysql-init/create-multiple-databases.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4fff1604..18b7be76 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,9 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # The released main layout the migration tests read (GD_RELEASED_MAIN). + - run: git fetch --no-tags --depth=1 origin main:refs/remotes/origin/main + - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 with: node-version: "26" diff --git a/docs/ghost-cli-replacement.md b/docs/ghost-cli-replacement.md index 8bbeb0b7..4d44228f 100644 --- a/docs/ghost-cli-replacement.md +++ b/docs/ghost-cli-replacement.md @@ -219,8 +219,9 @@ Gates the first stable tag and merging `next-docker` into `main`. Include the la on Linux, macOS and WSL2. Consolidate CI and qualify the actual minimum supported tools and image versions. Run fresh local/production install, optional-service variants, CLI migration, legacy stack update, Ghost upgrade/recovery, supervisor/Admin, and restore -scenarios. Refresh `tests/fixtures/released-main` from `main`'s final commit and rerun -`tests/e2e/migrate-main.sh` against it before merging. Include Linux runtime tests and macOS-compatible shell/configuration checks. +scenarios. Before merging, pin the migration tests' `GD_RELEASED_MAIN` +(default `origin/main`, in `manager/test/site.ts`, `tests/e2e/migrate-main.sh` and +the manager CI job) to `main`'s last commit, and rerun them. Include Linux runtime tests and macOS-compatible shell/configuration checks. Record the [compatibility baseline](architecture.md#compatibility): the exact metadata schema, backup format, launcher contract and bundle versions the stable release supports, and the obligations that hold for them from then on. diff --git a/manager/test/legacy.test.ts b/manager/test/legacy.test.ts index 316114d8..106af52a 100644 --- a/manager/test/legacy.test.ts +++ b/manager/test/legacy.test.ts @@ -7,9 +7,9 @@ import { join } from 'node:path'; import { describe, test } from 'node:test'; import { carryCaddyfile, fillEnvironment, type LegacyCaddyValues } from '../src/legacy/caddy.ts'; import { readLegacyEnv, refuseInterpolated, splitLegacyEnv } from '../src/legacy/config.ts'; -import { REPO } from './site.ts'; +import { releasedMain } from './site.ts'; -const MAIN = join(REPO, 'tests', 'fixtures', 'released-main'); +const MAIN = releasedMain(); const EXAMPLE = readFileSync(join(MAIN, 'caddy', 'Caddyfile.example'), 'utf8'); const VALUES: LegacyCaddyValues = { diff --git a/manager/test/migrate-main.test.ts b/manager/test/migrate-main.test.ts index 8c700aaf..aafe6697 100644 --- a/manager/test/migrate-main.test.ts +++ b/manager/test/migrate-main.test.ts @@ -11,6 +11,7 @@ import { imageApi, imageStack, REFERENCE, + releasedMain, REPO, resolvedProject, scriptSite, @@ -18,7 +19,7 @@ import { type ScriptedSite, } from './site.ts'; -const MAIN = join(REPO, 'tests', 'fixtures', 'released-main'); +const MAIN = releasedMain(); /** The Ghost-CLI-layout image main's site runs. */ const OLD_GHOST = `sha256:${'5'.repeat(64)}`; const CADDY = `caddy:2.10.2-alpine@sha256:${'6'.repeat(64)}`; diff --git a/manager/test/site.ts b/manager/test/site.ts index 60c63183..45d1afae 100644 --- a/manager/test/site.ts +++ b/manager/test/site.ts @@ -1,6 +1,7 @@ // A site directory for tests: the repository's own compose.yml and examples, // a `.env` written through the real encoder, and a scripted // `docker compose config` that answers what Compose would. +import { execFileSync } from 'node:child_process'; import { copyFileSync, cpSync, @@ -29,6 +30,42 @@ import { export const REPO = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +/** + * The released main layout's commit: `origin/main` until `next-docker` is + * merged into it, then pinned to main's last commit before that merge (S12 in + * docs/ghost-cli-replacement.md). Read from git, so it never drifts from main. + */ +export const RELEASED_MAIN = process.env.GD_RELEASED_MAIN ?? 'origin/main'; + +let releasedMainDir: string | undefined; + +/** + * The released main layout's files, checked out once per test process into a + * directory of their own, removed when the process exits. + */ +export function releasedMain(): string { + if (releasedMainDir !== undefined) { + return releasedMainDir; + } + let archive: Buffer; + try { + archive = execFileSync('git', ['-C', REPO, 'archive', '--format=tar', RELEASED_MAIN], { + stdio: ['ignore', 'pipe', 'pipe'], + maxBuffer: 256 * 1024 * 1024, + }); + } catch (error) { + throw new Error( + `the released main layout (${RELEASED_MAIN}) is not in this repository: ` + + `git fetch origin main, or set GD_RELEASED_MAIN. ${(error as Error).message}`, + ); + } + const dir = realpathSync(mkdtempSync(join(tmpdir(), 'gd-released-main-'))); + execFileSync('tar', ['-x', '-C', dir], { input: archive }); + process.on('exit', () => rmSync(dir, { recursive: true, force: true })); + releasedMainDir = dir; + return dir; +} + /** A bundle manifest, loosely typed so tests can break it. */ export type Manifest = Record; diff --git a/tests/e2e/migrate-main.sh b/tests/e2e/migrate-main.sh index 194b709f..0006f461 100755 --- a/tests/e2e/migrate-main.sh +++ b/tests/e2e/migrate-main.sh @@ -5,8 +5,9 @@ # tests/e2e/migrate-main.sh # # The installation is made as main's README made one: a git clone of the -# stack (tests/fixtures/released-main, committed, untagged, with a remote- -# tracking branch), `.env` and `caddy/Caddyfile` copied from the examples and +# repository at GD_RELEASED_MAIN (origin/main until next-docker is merged +# into it; then main's last commit before that merge, S12 in +# docs/ghost-cli-replacement.md), `.env` and `caddy/Caddyfile` copied from the examples and # edited, ActivityPub enabled, a custom route, a global options block, and a # compose.override.yml. It runs the official `ghost:6-alpine` image, whose # layout this release does not run. Then: @@ -33,6 +34,8 @@ IMAGE=$REGISTRY:$RELEASE DOMAIN=ghost-e2e.test HTTP_PORT=18080 HTTPS_PORT=18443 +MAIN_REPOSITORY=${GD_MAIN_REPOSITORY:-https://github.com/TryGhost/ghost-docker.git} +RELEASED_MAIN=${GD_RELEASED_MAIN:-origin/main} # shellcheck source=/dev/null source "$ROOT/tests/e2e/skip.sh" @@ -121,16 +124,11 @@ expect_status 0 ok "$IMAGE" step "An installation of the released main layout, as its README made one" -mkdir -p "$S" -cp -R "$ROOT/tests/fixtures/released-main/." "$S/" -rm "$S/README.md" -cp -R "$ROOT/tinybird" "$S/tinybird" -printf '.env\ndata\n' >"$S/.gitignore" -git -C "$S" init --quiet -git -C "$S" add -A -git -C "$S" -c user.name=e2e -c user.email=e2e@example.com commit --quiet -m 'main' -# As a clone has it: the commit is on a remote-tracking branch, and untagged. -git -C "$S" update-ref refs/remotes/origin/main HEAD +mkdir -p "$WORK/sites" +run git clone --quiet "$MAIN_REPOSITORY" "$S" +expect_status 0 +run git -C "$S" checkout --quiet --detach "$RELEASED_MAIN" +expect_status 0 cp "$S/.env.example" "$S/.env" sed -i.bak \ -e "s/^# COMPOSE_PROFILES=.*/COMPOSE_PROFILES=activitypub/" \ @@ -165,7 +163,7 @@ root_sql 'CREATE TABLE e2e_marker (note varchar(64)); INSERT INTO e2e_marker VAL compose_in exec -T ghost sh -c 'printf kept >/var/lib/ghost/content/images/e2e-marker.txt' [[ $(https /e2e-custom) == 'custom route kept' ]] || fail "main's custom route does not answer" "$(https /e2e-custom)" docker volume inspect "${project}_caddy_data" >/dev/null || fail "no ${project}_caddy_data volume" -ok "project $project, Ghost $version on ghost:6-alpine, with ActivityPub, a custom route and an override" +ok "$(git -C "$S" rev-parse --short HEAD) of main: project $project, Ghost $version on ghost:6-alpine, with ActivityPub, a custom route and an override" # --- Refused before anything changes ------------------------------------------------ diff --git a/tests/fixtures/released-main/.env.example b/tests/fixtures/released-main/.env.example deleted file mode 100644 index 9056225c..00000000 --- a/tests/fixtures/released-main/.env.example +++ /dev/null @@ -1,70 +0,0 @@ -# Use the below flags to enable the Analytics or ActivityPub containers as well -# COMPOSE_PROFILES=analytics,activitypub - -# Ghost domain -# Custom public domain Ghost will run on -DOMAIN=example.com - -# Ghost Admin domain -# If you have Ghost Admin setup on a separate domain uncomment the line below and add the domain -# You also need to uncomment the corresponding block in your Caddyfile -# ADMIN_DOMAIN= - -# Ghost ports -# Ports where Ghost will listen for HTTP traffic. -# Change these if the default ports are in use, or if Ghost is behind a reverse proxy. -HTTP_PORT=80 -HTTPS_PORT=443 - -# Database settings -# All database settings must not be changed once the database is initialised -DATABASE_ROOT_PASSWORD=reallysecurerootpassword -# DATABASE_USER=optionalusername -DATABASE_PASSWORD=ghostpassword - -# ActivityPub -# If you'd prefer to self-host ActivityPub yourself uncomment the line below -# ACTIVITYPUB_TARGET=activitypub:8080 - -# Tinybird configuration -# If you want to run Analytics, paste the output from `docker compose run --rm tinybird-login get-tokens` below -# TINYBIRD_API_URL=https://api.tinybird.co -# TINYBIRD_TRACKER_TOKEN=p.eyJxxxxx -# TINYBIRD_ADMIN_TOKEN=p.eyJxxxxx -# TINYBIRD_WORKSPACE_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - -# Shared secret for Ghost to authenticate Tinybird sync requests to Traffic Analytics -# for automation analytics. -# TINYBIRD_SYNC_AUTH=xxxxxxxx - -# Ghost configuration (https://ghost.org/docs/config/) - -# SMTP Email (https://ghost.org/docs/config/#mail) -# Transactional email is required for logins, account creation (staff invites), password resets and other features -# This is not related to bulk mail / newsletter sending -mail__transport=SMTP -mail__options__host=smtp.example.com -mail__options__port=465 -mail__options__secure=true -mail__options__auth__user=support@example.com -mail__options__auth__pass=1234567890 -mail__from="'Acme Support' " - -# Advanced customizations - -# Force Ghost version -# You should only do this if you need to pin a specific version -# The update commands won't work -# GHOST_VERSION=6-alpine - -# Port Ghost should listen on -# You should only need to edit this if you want to host -# multiple sites on the same server -# GHOST_PORT=2368 - -# Data locations -# Location to store uploaded data -UPLOAD_LOCATION=./data/ghost - -# Location for database data -MYSQL_DATA_LOCATION=./data/mysql diff --git a/tests/fixtures/released-main/README.md b/tests/fixtures/released-main/README.md deleted file mode 100644 index 14cf77a9..00000000 --- a/tests/fixtures/released-main/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# The released `main` layout - -The stack files of the layout released on `main`, as of commit `9871293`, for -the tests of migration `0001-compose-profiles` (`manager/src/legacy.ts`). An -installation of it is a git clone of these with an untracked `.env` and -`caddy/Caddyfile`, copied from the examples and edited. - -Refresh them from `main` if it changes before `next-docker` is merged into it: - -```bash -git archive origin/main compose.yml compose.ipv6.yml .env.example caddy mysql-init \ - | tar -x -C tests/fixtures/released-main -``` diff --git a/tests/fixtures/released-main/caddy/Caddyfile.example b/tests/fixtures/released-main/caddy/Caddyfile.example deleted file mode 100644 index 17b6e027..00000000 --- a/tests/fixtures/released-main/caddy/Caddyfile.example +++ /dev/null @@ -1,60 +0,0 @@ -{$DOMAIN} { - import snippets/Logging - - # Traffic Analytics service - import snippets/TrafficAnalytics - - # ActivityPub Service - import snippets/ActivityPub - - # Default proxy everything else to Ghost - handle { - reverse_proxy ghost:2368 - } - - # Optional: Enable gzip compression - encode gzip - - # Optional: Add security headers - import snippets/SecurityHeaders -} - -# Separate admin domains -# To use a separate domain for Ghost Admin uncomment the block below (recommended) -# {$ADMIN_DOMAIN} { -# import snippets/Logging -# -# # Traffic Analytics service -# import snippets/TrafficAnalytics -# -# # ActivityPub Service -# import snippets/ActivityPub -# -# # Default proxy everything else to Ghost -# handle { -# reverse_proxy ghost:2368 -# } -# -# # Optional: Enable gzip compression -# encode gzip -# -# # Optional: Add security headers -# import snippets/SecurityHeaders -# } - -# Redirect www -> root domain -# To redirect the www variant of your domain to the non-www variant uncomment the 4 lines below -# Note: You must have DNS setup correctly for both domains for this to work -# www.{$DOMAIN} { -# import snippets/Logging -# redir https://{$DOMAIN}{uri} -# } - -# Redirect root -> www domain -# To redirect the non-www variant of your domain to the www variant uncomment the 4 lines below and change CHANGE_ME to your root domain -# Note: You must have DNS setup correctly for both domains for this to work -# When using ActivityPub with a www. domain, you must enable this redirect for ActivityPub to work correctly -# CHANGE_ME { -# import snippets/Logging -# redir https://{$DOMAIN}{uri} -# } diff --git a/tests/fixtures/released-main/caddy/snippets/ActivityPub b/tests/fixtures/released-main/caddy/snippets/ActivityPub deleted file mode 100644 index e62d651e..00000000 --- a/tests/fixtures/released-main/caddy/snippets/ActivityPub +++ /dev/null @@ -1,13 +0,0 @@ -# ActivityPub -# Proxy activitypub requests /.ghost/activitypub/ -handle /.ghost/activitypub/* { - reverse_proxy {$ACTIVITYPUB_TARGET} -} - -handle /.well-known/webfinger { - reverse_proxy {$ACTIVITYPUB_TARGET} -} - -handle /.well-known/nodeinfo { - reverse_proxy {$ACTIVITYPUB_TARGET} -} diff --git a/tests/fixtures/released-main/caddy/snippets/Logging b/tests/fixtures/released-main/caddy/snippets/Logging deleted file mode 100644 index a0d6a666..00000000 --- a/tests/fixtures/released-main/caddy/snippets/Logging +++ /dev/null @@ -1,6 +0,0 @@ -# Log all requests -log { - output stdout - format console - level INFO -} diff --git a/tests/fixtures/released-main/caddy/snippets/SecurityHeaders b/tests/fixtures/released-main/caddy/snippets/SecurityHeaders deleted file mode 100644 index a0b0d4ef..00000000 --- a/tests/fixtures/released-main/caddy/snippets/SecurityHeaders +++ /dev/null @@ -1,12 +0,0 @@ -header { - # Enable HSTS - Strict-Transport-Security max-age=31536000; - # Enable XSS protection - X-XSS-Protection "1; mode=block" - # Prevent MIME sniffing - X-Content-Type-Options nosniff - # Referrer policy - Referrer-Policy strict-origin-when-cross-origin - # Prevent embedding in external iframes - Content-Security-Policy "frame-ancestors 'self' {$ADMIN_DOMAIN:}" -} diff --git a/tests/fixtures/released-main/caddy/snippets/TrafficAnalytics b/tests/fixtures/released-main/caddy/snippets/TrafficAnalytics deleted file mode 100644 index 53a81f2c..00000000 --- a/tests/fixtures/released-main/caddy/snippets/TrafficAnalytics +++ /dev/null @@ -1,6 +0,0 @@ -# Proxy analytics requests with any prefix (e.g. /.ghost/analytics/ or /blog/.ghost/analytics/) -@analytics_paths path_regexp analytics_match ^(.*)/\.ghost/analytics(.*)$ -handle @analytics_paths { - rewrite * {re.analytics_match.2} - reverse_proxy traffic-analytics:3000 -} diff --git a/tests/fixtures/released-main/compose.ipv6.yml b/tests/fixtures/released-main/compose.ipv6.yml deleted file mode 100644 index e2385f83..00000000 --- a/tests/fixtures/released-main/compose.ipv6.yml +++ /dev/null @@ -1,6 +0,0 @@ -networks: - ghost_network: - enable_ipv6: true - ipam: - config: - - subnet: fdc1:f5e4:6668::/64 diff --git a/tests/fixtures/released-main/compose.yml b/tests/fixtures/released-main/compose.yml deleted file mode 100644 index e56ea3c9..00000000 --- a/tests/fixtures/released-main/compose.yml +++ /dev/null @@ -1,209 +0,0 @@ ---- -# yaml-language-server: $schema=https://raw.githubusercontent.com/compose-spec/compose-spec/main/schema/compose-spec.json -services: - caddy: - image: caddy:2.10.2-alpine@sha256:953131cfea8e12bfe1c631a36308e9660e4389f0c3dfb3be957044d3ac92d446 - restart: always - ports: - - "${HTTP_PORT:-80}:80" - - "${HTTPS_PORT:-443}:443" - environment: - DOMAIN: ${DOMAIN:?DOMAIN environment variable is required} - ADMIN_DOMAIN: ${ADMIN_DOMAIN:-} - ACTIVITYPUB_TARGET: ${ACTIVITYPUB_TARGET:-https://ap.ghost.org} - volumes: - - ./caddy:/etc/caddy - - caddy_data:/data - - caddy_config:/config - depends_on: - - ghost - networks: - - ghost_network - - ghost: - # Do not alter this without updating the Tinybird Sync container as well - image: ghost:${GHOST_VERSION:-6-alpine} - restart: always - # This is required to import current config when migrating - env_file: - - .env - environment: - NODE_ENV: production - url: https://${DOMAIN:?DOMAIN environment variable is required} - admin__url: ${ADMIN_DOMAIN:+https://${ADMIN_DOMAIN}} - database__client: mysql - database__connection__host: db - database__connection__user: ${DATABASE_USER:-ghost} - database__connection__password: ${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required} - database__connection__database: ghost - tinybird__tracker__endpoint: https://${DOMAIN:?DOMAIN environment variable is required}/.ghost/analytics/api/v1/page_hit - tinybird__adminToken: ${TINYBIRD_ADMIN_TOKEN:-} - tinybird__workspaceId: ${TINYBIRD_WORKSPACE_ID:-} - tinybird__sync_auth_key: ${TINYBIRD_SYNC_AUTH:-} - tinybird__tracker__datasource: analytics_events - tinybird__stats__endpoint: ${TINYBIRD_API_URL:-https://api.tinybird.co} - volumes: - - ${UPLOAD_LOCATION:-./data/ghost}:/var/lib/ghost/content - depends_on: - db: - condition: service_healthy - tinybird-sync: - condition: service_completed_successfully - required: false - tinybird-deploy: - condition: service_completed_successfully - required: false - activitypub: - condition: service_started - required: false - networks: - - ghost_network - - db: - image: mysql:8.0.44@sha256:f37951fc3753a6a22d6c7bf6978c5e5fefcf6f31814d98c582524f98eae52b21 - restart: always - expose: - - "3306" - environment: - MYSQL_ROOT_PASSWORD: ${DATABASE_ROOT_PASSWORD:?DATABASE_ROOT_PASSWORD environment variable is required} - MYSQL_USER: ${DATABASE_USER:-ghost} - MYSQL_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required} - MYSQL_DATABASE: ghost - MYSQL_MULTIPLE_DATABASES: activitypub - volumes: - - ${MYSQL_DATA_LOCATION:-./data/mysql}:/var/lib/mysql - - ./mysql-init:/docker-entrypoint-initdb.d - healthcheck: - test: mysqladmin ping -p$$MYSQL_ROOT_PASSWORD -h 127.0.0.1 - interval: 1s - start_period: 30s - start_interval: 10s - retries: 120 - networks: - - ghost_network - - traffic-analytics: - image: ghost/traffic-analytics:1.0.461@sha256:6d9324fb1fe43037338c7bd5ef5ff04390b16dad449cfc48831ebb00ddae9652 - restart: always - expose: - - "3000" - volumes: - - traffic_analytics_data:/data - environment: - NODE_ENV: production - PROXY_TARGET: ${TINYBIRD_API_URL:-https://api.tinybird.co}/v0/events - SALT_STORE_TYPE: ${SALT_STORE_TYPE:-file} - SALT_STORE_FILE_PATH: /data/salts.json - TINYBIRD_TRACKER_TOKEN: ${TINYBIRD_TRACKER_TOKEN:-} - TINYBIRD_SYNC_AUTH: ${TINYBIRD_SYNC_AUTH:-} - LOG_LEVEL: debug - profiles: [analytics] - networks: - - ghost_network - - activitypub: - image: ghcr.io/tryghost/activitypub:1.2.14@sha256:a607c8cd82d7d84eabec7abdb5c98e2788605572a0fa79ba75a872787e0084bc - restart: always - expose: - - "8080" - volumes: - - ${UPLOAD_LOCATION:-./data/ghost}:/opt/activitypub/content - environment: - # See https://github.com/TryGhost/ActivityPub/blob/main/docs/env-vars.md - NODE_ENV: production - MYSQL_HOST: db - MYSQL_USER: ${DATABASE_USER:-ghost} - MYSQL_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required} - MYSQL_DATABASE: activitypub - LOCAL_STORAGE_PATH: /opt/activitypub/content/images/activitypub - LOCAL_STORAGE_HOSTING_URL: https://${DOMAIN}/content/images/activitypub - depends_on: - db: - condition: service_healthy - activitypub-migrate: - condition: service_completed_successfully - profiles: [activitypub] - networks: - - ghost_network - - # Supporting Services - - tinybird-login: - build: - context: ./tinybird - dockerfile: Dockerfile - working_dir: /home/tinybird - command: /usr/local/bin/tinybird-login - volumes: - - tinybird_home:/home/tinybird - - tinybird_files:/data/tinybird - profiles: [analytics] - networks: - - ghost_network - tty: false - restart: no - - tinybird-sync: - # Do not alter this without updating the Ghost container as well - image: ghost:${GHOST_VERSION:-6-alpine} - command: > - sh -c " - if [ -d /var/lib/ghost/current/core/server/data/tinybird ]; then - rm -rf /data/tinybird/*; - cp -rf /var/lib/ghost/current/core/server/data/tinybird/* /data/tinybird/; - echo 'Tinybird files synced into shared volume.'; - else - echo 'Tinybird source directory not found.'; - fi - " - volumes: - - tinybird_files:/data/tinybird - depends_on: - tinybird-login: - condition: service_completed_successfully - networks: - - ghost_network - profiles: [analytics] - restart: no - - tinybird-deploy: - build: - context: ./tinybird - dockerfile: Dockerfile - working_dir: /data/tinybird - command: > - sh -c " - tb-wrapper --cloud deploy - " - volumes: - - tinybird_home:/home/tinybird - - tinybird_files:/data/tinybird - depends_on: - tinybird-sync: - condition: service_completed_successfully - profiles: [analytics] - networks: - - ghost_network - tty: true - - activitypub-migrate: - image: ghcr.io/tryghost/activitypub-migrations:1.2.14@sha256:c116a15f9c8a94542440fadc17bc06c15eeee6e3a61ff2731e3b6e2ea91f7356 - environment: - MYSQL_DB: mysql://${DATABASE_USER:-ghost}:${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required}@tcp(db:3306)/activitypub - networks: - - ghost_network - depends_on: - db: - condition: service_healthy - profiles: [activitypub] - restart: no - -volumes: - caddy_data: - caddy_config: - tinybird_files: - tinybird_home: - traffic_analytics_data: - -networks: - ghost_network: diff --git a/tests/fixtures/released-main/mysql-init/create-multiple-databases.sh b/tests/fixtures/released-main/mysql-init/create-multiple-databases.sh deleted file mode 100755 index 6b0d32ea..00000000 --- a/tests/fixtures/released-main/mysql-init/create-multiple-databases.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e -set -u - -if [ -n "$MYSQL_MULTIPLE_DATABASES" ]; then - echo "Creating multiple databases: $MYSQL_MULTIPLE_DATABASES" - - for db in $(echo "$MYSQL_MULTIPLE_DATABASES" | tr ',' ' '); do - echo "Creating database: $db" - mysql -u root -p"$MYSQL_ROOT_PASSWORD" <<-EOSQL - CREATE DATABASE IF NOT EXISTS \`$db\`; - GRANT ALL ON \`$db\`.* TO '$MYSQL_USER'@'%'; -EOSQL - done - - echo "Multiple databases created" -fi From af4b92a8e3b72c465c91db72cfab4253230a6541 Mon Sep 17 00:00:00 2001 From: Austin Burdine Date: Fri, 9 Oct 2026 16:36:33 -0400 Subject: [PATCH 5/5] Fixed a concurrent migration's cleanup, stranded mounts and override settings ref https://linear.app/ghost/issue/PLA-482 Three problems review found in the migration from the released main layout: - The lock was taken only after preparation, which stages into and then removes .ghost-docker-update/. A second run overlapping the first could remove the first's snapshot, so a later failure in the first could not put the site's files back. The lock is now held from before anything is staged, --check included. - An override mounting into /var/lib/ghost (main's image's install and content path) survived the move to an image that keeps content in /home/ghost/content: the uploads it held became invisible to Ghost and to the backup, and Compose accepted it. It is now refused, naming the mount. - The new layout was first resolved with only the generated settings, so an override requiring an existing .env setting (`${SMTP_HOST:?}`) failed to resolve. Draft resolution now has every setting .env had. - manager/src/legacy.ts: the three fixes. - manager/test/migrate-main.test.ts: a test for each. - docs/install.md: overrides resolve with .env; /var/lib/ghost mounts are refused. --- docs/install.md | 2 +- manager/src/legacy.ts | 69 ++++++++++++++++++++-------- manager/test/migrate-main.test.ts | 76 ++++++++++++++++++++++++++++++- 3 files changed, 125 insertions(+), 22 deletions(-) diff --git a/docs/install.md b/docs/install.md index 30f9fbf6..138f7bfe 100644 --- a/docs/install.md +++ b/docs/install.md @@ -687,7 +687,7 @@ resolve, if any of it cannot be carried over: | Ghost's configuration in `.env` (`mail__*`, `labs__*`, anything Ghost read) | Moved to `ghost.env`, with the values Ghost actually received. Keys the container sets itself (`url`, `database__*`) are listed and left out, as they were overridden on main too | | `ghost:6-alpine` | The `next` image of **exactly the running version**, pinned by digest (`GHOST_IMAGE_REF`). Ghost 6.61.0 is the first with one: upgrade an older Ghost on main first (`docker compose pull ghost && docker compose up -d`) | | `caddy/Caddyfile` | Kept as `caddy/Caddyfile.local`, and carried into `caddy/sites/site.caddy` as written: `{$DOMAIN}`, `{$ADMIN_DOMAIN}` and `{$ACTIVITYPUB_TARGET}` filled in, and `import snippets/...` pointed at main's snippets, kept in `caddy/sites/legacy-snippets/`. A leading global options block moves to `caddy/global/legacy.caddy`. Caddy loads the result before anything changes; if it does not load, nothing is changed | -| `compose.override.yml`, `GD_COMPOSE_OVERRIDES` | Kept, and resolved with the new layout before anything changes | +| `compose.override.yml`, `GD_COMPOSE_OVERRIDES` | Kept, and resolved with the new layout and every `.env` setting before anything changes. A mount into `/var/lib/ghost`, where main's Ghost image kept its files, is refused: this layout's image keeps them in `/home/ghost/content`, and a backup holds content only in `data/ghost` | | Stack files (`compose.yml`, snippets, `mysql-init/`) | The release's. Git must show them unedited: a change in the work tree or in a commit no remote has stops the migration. Move such changes into `compose.override.yml` or a `.caddy` file of your own afterwards | Then it keeps a copy of every file it writes in `.ghost-docker-update/`, stops diff --git a/manager/src/legacy.ts b/manager/src/legacy.ts index 964724c2..bbfeef4d 100644 --- a/manager/src/legacy.ts +++ b/manager/src/legacy.ts @@ -114,6 +114,8 @@ const PULL_MS = 30 * 60 * 1000; /** The optional profiles `main` had. */ const MAIN_PROFILES = ['analytics', 'activitypub']; const HOSTED_ACTIVITYPUB = 'https://ap.ghost.org'; +/** Where main's Ghost-CLI-layout image is installed, its content included. */ +const MAIN_GHOST_INSTALL = '/var/lib/ghost'; /** Where preflight stages the new `.env` and Caddy files, inside the site so the daemon sees them. */ const STAGED = 'staged'; @@ -183,26 +185,25 @@ export async function migrateReleasedMain( ' curl -fsSL https://docker.ghost.org/install.sh | bash -s -- self-update', ); } - if (existsSync(join(dir, UPDATE_DIR))) { - refuse( - `${join(dir, UPDATE_DIR)} is left from a migration or update that did not finish, and holds\n` + - ' the files it would have put back. Once the site is as it should be, remove it and run this again.', - ); - } - - const staged = join(dir, UPDATE_DIR, STAGED); - let plan: Plan; - try { - plan = await prepare(io, context, site, requested, staged); - } finally { - rmSync(join(dir, UPDATE_DIR), { recursive: true, force: true }); - } - if (check) { - return report(plan); - } + // Held from before anything is staged, --check included: the staging + // directory is the one the snapshot is kept in, and a second run must + // never remove another's. const lock = acquireLock(dir, `migration ${MIGRATION} from ${FROM}`); try { - return await apply(plan); + if (existsSync(join(dir, UPDATE_DIR))) { + refuse( + `${join(dir, UPDATE_DIR)} is left from a migration or update that did not finish, and holds\n` + + ' the files it would have put back. Once the site is as it should be, remove it and run this again.', + ); + } + const staged = join(dir, UPDATE_DIR, STAGED); + let plan: Plan; + try { + plan = await prepare(io, context, site, requested, staged); + } finally { + rmSync(join(dir, UPDATE_DIR), { recursive: true, force: true }); + } + return check ? report(plan) : await apply(plan); } finally { lock.release(); } @@ -361,7 +362,19 @@ async function prepare( files: [join(stack, COMPOSE_FILE), ...before.files.slice(1)], envFile: join(staged, ENV_FILE), }; - atomicWrite(inputs.envFile, env.serializeAll(generated), PRIVATE); + // With every setting .env had, so an override requiring one of them + // (`${SMTP_HOST:?...}`) resolves as it does now. + const generatedKeys = new Set(generated.map(([key]) => key)); + atomicWrite( + inputs.envFile, + env.serializeAll([ + ...generated, + ...legacy.keys + .filter((key) => !generatedKeys.has(key)) + .map((key): [string, string] => [key, old[key]!]), + ]), + PRIVATE, + ); const draft = await composeConfig(io, dir, inputs); const newVariables = await composeVariables(io, dir, inputs); if (!draft.ok || newVariables === null) { @@ -381,7 +394,7 @@ async function prepare( refuseInterpolated(text, new Set(legacy.keys.filter(isOperatorKey))); const split = splitLegacyEnv({ legacy, - generated: new Set(generated.map(([key]) => key)), + generated: generatedKeys, isOperatorKey, isInterpolated: (key) => newVariables.has(key), container: new Set(Object.keys(draft.project.services.ghost?.environment ?? {})), @@ -408,6 +421,22 @@ async function prepare( ` ${ghost.reference}. Remove the override that sets its image, then run this again.`, ); } + // Where main's Ghost image was installed. This layout's image never looks + // there, so whatever an override mounts there would silently be lost to + // Ghost, and to the backup. + const stranded = (after.services.ghost?.mounts ?? []).filter( + (mount) => + mount.target === MAIN_GHOST_INSTALL || + mount.target.startsWith(`${MAIN_GHOST_INSTALL}/`), + ); + if (stranded.length > 0) { + refuse( + `the site's overrides mount ${stranded.map((mount) => `${mount.source} at ${mount.target}`).join(', ')} in the\n` + + ` ghost service. That is where main's Ghost image kept its files; this layout's keeps them in\n` + + ` ${ghost.contentPath}, and a backup holds content only in ./data/ghost. Move what the mount holds\n` + + ' into data/ghost (or remove it), remove the mount from the override, then run this again.', + ); + } const draftSite = siteFacts(dir, { get: (key) => env.toRecord(envText)[key], }); diff --git a/manager/test/migrate-main.test.ts b/manager/test/migrate-main.test.ts index aafe6697..a95f7384 100644 --- a/manager/test/migrate-main.test.ts +++ b/manager/test/migrate-main.test.ts @@ -2,10 +2,11 @@ // 0001-compose-profiles, against a scripted daemon, Compose and git. What it // does on a real host is tests/e2e/migrate-main.sh; this is what it decides. import assert from 'node:assert/strict'; -import { cpSync, existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import { basename, join } from 'node:path'; import { afterEach, beforeEach, describe, test } from 'node:test'; import * as env from '../src/env.ts'; +import { acquireLock } from '../src/lock.ts'; import { failed, harness, json, ok, type Harness, type ProgramResult } from './helpers.ts'; import { imageApi, @@ -53,6 +54,8 @@ let committed: string[]; /** What `caddy validate` answers. */ let caddyValidates: { status: number; stderr?: string }; const validated: string[] = []; +/** A mount the operator's override adds to the ghost service, in either layout. */ +let overrideMount: { type: string; source: string; target: string } | null; const readSite = (file: string) => readFileSync(join(h.dir, file), 'utf8'); const envOf = (file: string) => env.toRecord(readSite(file)); @@ -79,6 +82,15 @@ function files(dir = h.dir, prefix = ''): Record { function config(_args: string[], envFile?: string): ProgramResult { const values = env.toRecord(readFileSync(envFile ?? join(h.dir, '.env'), 'utf8')); const legacy = values.URL === undefined; + // What the override requires, as Compose interpolates `${NAME:?...}`. + const override = existsSync(join(h.dir, 'compose.override.yml')) + ? readSite('compose.override.yml') + : ''; + for (const [, name] of override.matchAll(/\$\{([A-Za-z_]\w*):\?/g)) { + if (!values[name!]) { + return failed(1, `required variable ${name} is missing a value`); + } + } const environment: Record = legacy ? // main's env_file is the whole of .env. { ...values, url: `https://${values.DOMAIN}`, NODE_ENV: 'production' } @@ -106,6 +118,9 @@ function config(_args: string[], envFile?: string): ProgramResult { if (legacy) { resolved.services.ghost.volumes[0].target = '/var/lib/ghost/content'; } + if (overrideMount !== null) { + resolved.services.ghost.volumes.push(overrideMount); + } return ok(JSON.stringify(resolved)); } @@ -128,6 +143,7 @@ beforeEach(() => { edited = []; committed = []; caddyValidates = { status: 0 }; + overrideMount = null; validated.length = 0; const images = imageApi({ ghost: { '6.67.0-next-alpine': '6.67.0' } }); @@ -379,6 +395,64 @@ describe('migrating the released main layout', () => { ); }); + test('a second run waits for no one: another holding the lock is refused, its snapshot kept', async () => { + // Another migration, part-way through: its lock and its snapshot. + const other = acquireLock( + h.dir, + 'migration 0001-compose-profiles from the released main layout', + ); + mkdirSync(join(h.dir, '.ghost-docker-update', 'files'), { recursive: true }); + writeFileSync(join(h.dir, '.ghost-docker-update', 'files', '.env'), OLD_ENV); + try { + for (const args of [[], ['--check']]) { + const result = await migrate(...args); + assert.equal(result.code, 1); + // Refused by the lock, before it looks at, stages or removes anything. + assert.match( + result.stderr, + /\.ghost-docker\.lock is held by migration 0001-compose-profiles/, + ); + assert.ok(!h.calls.some((call) => call[0] === 'git'), 'it began preparing'); + assert.equal( + readSite('.ghost-docker-update/files/.env'), + OLD_ENV, + 'the other run’s snapshot was touched', + ); + } + } finally { + other.release(); + } + }); + + test("an override mounting into main's Ghost install is refused: this layout's Ghost never looks there", async () => { + overrideMount = { + type: 'bind', + source: join(h.dir, 'images'), + target: '/var/lib/ghost/content/images', + }; + const before = files(); + const result = await migrate(); + assert.equal(result.code, 1); + assert.match( + result.stderr, + /at \/var\/lib\/ghost\/content\/images in the\n\s+ghost service/, + ); + assert.match(result.stderr, /Nothing has been changed/); + assert.deepEqual(files(), before); + }); + + test('an override requiring a setting .env has resolves with the new layout too', async () => { + writeFileSync( + join(h.dir, 'compose.override.yml'), + 'services:\n ghost:\n environment:\n mail__options__host: ${SMTP_HOST:?SMTP_HOST is required}\n', + ); + writeFileSync(join(h.dir, '.env'), `${OLD_ENV}SMTP_HOST=smtp.example.com\n`); + const result = await migrate(); + assert.equal(result.code, 0, result.stderr); + assert.equal(envOf('.env').SMTP_HOST, 'smtp.example.com'); + assert.equal(envOf('ghost.env').SMTP_HOST, undefined); + }); + test('a site that is not running is refused: its Ghost version is unknown', async () => { h.daemon.containers = []; const result = await migrate();