Robustness: null-deref, div-by-zero and control-flow guards (#1380) #2443

Workflow file for this run

	name: E2E Tests

	on:
	push:
	branches:
	- main
	pull_request:
	branches:
	- "**"

	jobs:
	cypress:
	runs-on: ubuntu-latest
	timeout-minutes: 45 # Increased for setup wizard + Stripe test clock advancement
	strategy:
	matrix:
	php: ["8.5"]
	browser: ["chrome"] # Start with Chrome only for reliability

	services:
	mailpit:
	image: ghcr.io/axllent/mailpit@sha256:3bd7c2f2696deb35a4780d152b404dceec99cb041b942c0877b3b22384714f85
	ports:
	- 1025:1025
	- 8025:8025
	options: >-
	--health-cmd="wget --spider -q http://localhost:8025 \|\| exit 1"
	--health-interval=2s
	--health-timeout=2s
	--health-retries=10

	steps:
	- name: Checkout code
	uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

	- name: Setup Node.js
	uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
	with:
	node-version: 18

	- name: Cache NPM dependencies
	uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3
	with:
	path: ~/.npm
	key: ${{ runner.os }}-npm-cache

	- name: Install NPM dependencies
	run: npm ci

	- name: Cache Composer dependencies
	uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3
	with:
	path: vendor
	key: ${{ runner.os }}-composer-cache

	- name: Install Composer dependencies
	run: composer install

	- name: Set PHP version and filter missing plugins for wp-env
	run: \|
	node -e "
	const fs = require('fs');
	const path = require('path');
	const config = JSON.parse(fs.readFileSync('.wp-env.json', 'utf8'));
	// phpVersion must be a top-level wp-env option, not inside 'config'
	// (config maps to wp-config.php constants; phpVersion there is a no-op).
	const override = { phpVersion: '${{ matrix.php }}', env: {} };
	for (const [envName, envConfig] of Object.entries(config.env \|\| {})) {
	if (envConfig.plugins) {
	const existing = envConfig.plugins.filter(p => {
	const resolved = path.resolve(p);
	return fs.existsSync(resolved);
	});
	override.env[envName] = { plugins: existing };
	}
	}
	fs.writeFileSync('.wp-env.override.json', JSON.stringify(override, null, 2));
	console.log('Override:', JSON.stringify(override, null, 2));
	"

	- name: Start WordPress Test Environment
	run: \|
	# Retry up to 3 times — wp-env occasionally gets a transient 502 from
	# the wordpress.org CDN when downloading the core zip.
	set +e
	for attempt in 1 2 3; do
	echo "Attempt $attempt: starting WordPress test environment..."
	npm run env:start:test
	EXIT_CODE=$?
	if [ $EXIT_CODE -eq 0 ]; then
	echo "✅ WordPress environment started (attempt $attempt)"
	break
	fi
	echo "⚠️ Attempt $attempt failed (exit $EXIT_CODE)"
	if [ $attempt -lt 3 ]; then
	echo "Stopping any partial environment and retrying in 30s..."
	npm run env:stop 2>/dev/null \|\| true
	sleep 30
	else
	echo "❌ All 3 attempts failed — giving up"
	exit $EXIT_CODE
	fi
	done

	- name: Add mapped SSO host entry
	run: \|
	echo "127.0.0.1 sso-test.ultimate-multisite.test" \| sudo tee -a /etc/hosts
	getent hosts sso-test.ultimate-multisite.test

	- name: Wait for WordPress to be ready
	run: \|
	for i in {1..60}; do
	if curl -s http://localhost:8889 \| grep -q "WordPress"; then
	echo "WordPress is ready";
	break;
	fi
	echo "Waiting for WordPress... ($i/60)";
	sleep 5;
	done
	echo "Final check:"
	curl -s http://localhost:8889 \|\| echo "WordPress not responding"

	- name: Network-activate plugin in tests environment
	run: \|
	echo "=== Network-activating Ultimate Multisite plugin ==="
	PLUGIN_SLUG=$(basename "$PWD")
	echo "Plugin slug: $PLUGIN_SLUG"

	# Network-activate so wu_save_setting() and other functions
	# are fully bootstrapped for all wp eval / eval-file calls.
	npx wp-env run tests-cli wp plugin activate "$PLUGIN_SLUG" --network \|\| \
	npx wp-env run tests-cli wp plugin activate ultimate-multisite --network \|\| \
	echo "⚠️ Could not network-activate by directory name, listing plugins..."

	echo "Plugin list after activation attempt:"
	npx wp-env run tests-cli wp plugin list --status=active --format=table

	echo "Verifying wu_save_setting() is available:"
	npx wp-env run tests-cli wp eval "var_dump(function_exists('wu_save_setting'));" \|\| true

	- name: Comprehensive WordPress Debug
	run: \|
	echo "=== WordPress Environment Debug ==="
	echo "1. Basic URL Tests:"
	curl -I http://localhost:8889 \|\| echo "❌ WordPress not responding"
	curl -I http://localhost:8889/wp-admin/ \|\| echo "❌ WP Admin not responding"
	curl -I http://localhost:8889/wp-admin/network/ \|\| echo "❌ Network admin not responding"

	echo -e "\n2. WordPress Content Test:"
	curl -s http://localhost:8889 \| head -20 \| grep -E "(WordPress\|wp-)" \|\| echo "❌ No WordPress indicators found"

	echo -e "\n3. Plugin Status Check:"
	curl -s "http://localhost:8889/wp-admin/network/plugins.php" \| grep -i "multisite-ultimate" \|\| echo "❌ Plugin not found in network admin"

	echo -e "\n4. Database Connection Test:"
	curl -s "http://localhost:8889/wp-admin/network/" \| grep -E "(Database\|MySQL\|connection)" \|\| echo "✅ No database errors visible"

	echo -e "\n5. Multisite Status:"
	curl -s http://localhost:8889 \| grep -i "multisite\\|network" \|\| echo "ℹ️ No multisite indicators in homepage"

	echo -e "\n6. Error Log Check:"
	docker logs $(docker ps -q --filter "name=tests-wordpress") 2>&1 \| tail -20 \|\| echo "ℹ️ Could not get container logs"

	echo -e "\n7. File System Check:"
	ls -la $(pwd) \|\| echo "❌ Could not list current directory"
	ls -la $(pwd)/multisite-ultimate.php 2>/dev/null && echo "✅ Main plugin file exists" \|\| echo "❌ Main plugin file not found"

	- name: Pre-Test Environment Verification
	run: \|
	echo "=== Pre-Test Verification ==="
	echo "1. Cypress Config Check:"
	ls -la cypress.config.test.js && echo "✅ Config file exists" \|\| echo "❌ Config file missing"

	echo "2. Test Files Check:"
	ls -la tests/e2e/cypress/integration/ && echo "✅ Test directory exists" \|\| echo "❌ Test directory missing"
	find tests/e2e/cypress/integration/ -name "*.spec.js" \| wc -l \| xargs echo "Test files found:"

	echo "3. Support Files Check:"
	ls -la tests/e2e/cypress/support/commands/ && echo "✅ Commands directory exists" \|\| echo "❌ Commands missing"

	echo "4. Cypress Environment Variables:"
	echo "Base URL: http://localhost:8889"
	cat cypress.env.json 2>/dev/null \|\| echo "❌ cypress.env.json not found"

	- name: Run Setup Test (Must Run First)
	id: setup-test
	run: \|
	echo "=== Starting Setup Test ==="
	npx cypress run \
	--config-file cypress.config.test.js \
	--spec "tests/e2e/cypress/integration/000-setup.spec.js" \
	--browser ${{ matrix.browser }}

	- name: Dump container logs on setup failure
	if: failure() && steps.setup-test.outcome == 'failure'
	run: \|
	echo "=== Container logs (last 100 lines) ==="
	docker logs $(docker ps -q --filter "name=tests-wordpress") 2>&1 \| tail -100 \|\| echo "Could not get container logs"
	echo "=== PHP error log ==="
	npx wp-env run tests-cli cat /var/www/html/wp-content/debug.log 2>&1 \| tail -100 \|\| echo "No debug.log found"

	- name: Run Sanity Tests (login, mail)
	id: sanity-tests
	run: \|
	set +e
	echo "=== Starting Sanity Tests ==="

	# Smoke tests that don't depend on Ultimate Multisite plugin state —
	# verify the WordPress test environment, admin login, and Mailpit
	# service are all functional before exercising plugin features.
	SANITY_TESTS=(
	"tests/e2e/cypress/integration/login.spec.js"
	"tests/e2e/cypress/integration/mail.spec.js"
	)

	TOTAL_FAILURES=0

	for TEST_SPEC in "${SANITY_TESTS[@]}"; do
	echo "Running: $TEST_SPEC"

	npx cypress run \
	--config-file cypress.config.test.js \
	--spec "$TEST_SPEC" \
	--browser ${{ matrix.browser }}

	CYPRESS_EXIT_CODE=$?

	if [ $CYPRESS_EXIT_CODE -eq 0 ]; then
	echo "✅ $TEST_SPEC passed"
	else
	echo "❌ $TEST_SPEC failed with exit code $CYPRESS_EXIT_CODE"
	TOTAL_FAILURES=$((TOTAL_FAILURES + 1))
	fi
	done

	if [ $TOTAL_FAILURES -gt 0 ]; then
	echo "❌ $TOTAL_FAILURES sanity test(s) failed"
	exit 1
	fi

	echo "✅ All sanity tests passed!"

	- name: Run Wizard Test (Setup Wizard UI)
	id: wizard-test
	run: \|
	echo "=== Starting Setup Wizard UI Test ==="

	# wizard.spec.js drives the actual Setup Wizard form submission path
	# (handle_save_settings -> Settings::save_settings -> Field::set_value),
	# which 000-setup.spec.js bypasses by calling the installer directly.
	# The wizard spec clears NETWORK_OPTION_SETUP_FINISHED itself before
	# running, then re-marks setup complete at the end — safe to run after
	# 000-setup. This regression-guards the addslashes/TypeError class of
	# bugs that only surface when the wizard form actually POSTs.
	npx cypress run \
	--config-file cypress.config.test.js \
	--spec "tests/e2e/cypress/integration/wizard.spec.js" \
	--browser ${{ matrix.browser }}

	- name: Reset custom login page (post-wizard cleanup)
	id: reset-custom-login
	run: \|
	echo "=== Disabling custom login page after wizard ==="
	# The wizard's "Default Content" step creates a Login page and points
	# Ultimate Multisite's custom-login feature at it, which redirects
	# /wp-login.php -> /login/. The custom page has no #rememberme
	# checkbox, so subsequent specs that use cy.loginByForm() fail in
	# their session-setup hook. 000-setup.spec.js calls this same
	# fixture defensively in its before() hook for the same reason.
	PLUGIN_SLUG=$(basename "$PWD")
	npx wp-env run tests-cli wp eval-file \
	"/var/www/html/wp-content/plugins/$PLUGIN_SLUG/tests/e2e/cypress/fixtures/setup-disable-custom-login.php" \|\| \
	npx wp-env run tests-cli wp eval-file \
	"/var/www/html/wp-content/plugins/ultimate-multisite/tests/e2e/cypress/fixtures/setup-disable-custom-login.php"

	- name: Run Regression & UI Tests (After Setup)
	id: regression-tests
	run: \|
	set +e
	echo "=== Starting Regression & UI Tests ==="

	# High-value regression guards for previously-fixed bugs and UI
	# error-handling. All are self-contained (WP-CLI fixtures or pure
	# browser interaction) and need no external credentials.
	REGRESSION_TESTS=(
	"tests/e2e/cypress/integration/011-password-reset-subsite-domain.spec.js"
	"tests/e2e/cypress/integration/030-modal-form-error-handling.spec.js"
	"tests/e2e/cypress/integration/050-password-strength-enforcement.spec.js"
	"tests/e2e/cypress/integration/066-sso-bootstrap-race.spec.js"
	)

	TOTAL_FAILURES=0

	for TEST_SPEC in "${REGRESSION_TESTS[@]}"; do
	echo "Running: $TEST_SPEC"

	npx cypress run \
	--config-file cypress.config.test.js \
	--spec "$TEST_SPEC" \
	--browser ${{ matrix.browser }}

	CYPRESS_EXIT_CODE=$?

	if [ $CYPRESS_EXIT_CODE -eq 0 ]; then
	echo "✅ $TEST_SPEC passed"
	else
	echo "❌ $TEST_SPEC failed with exit code $CYPRESS_EXIT_CODE"
	TOTAL_FAILURES=$((TOTAL_FAILURES + 1))
	fi
	done

	if [ $TOTAL_FAILURES -gt 0 ]; then
	echo "❌ $TOTAL_FAILURES regression test(s) failed"
	exit 1
	fi

	echo "✅ All regression tests passed!"

	- name: Run Checkout Tests (After Setup)
	id: checkout-tests
	run: \|
	set +e
	echo "=== Starting Checkout Tests ==="

	# TODO(#1317): re-enable 010-manual-checkout-flow.spec.js once
	# the manual-gateway regression is fixed. The spec has never been
	# green on main since PR #1280 (commit 57176437) wired it into CI:
	# the form submission on /register/ does not reach `status=done` and
	# no payment row is created (`um_payment_status: 'no-payments'`).
	# Suspected root cause: 000-setup's `Test Plan` is recurring by
	# default (Product model default `recurring=1`), and
	# `Manual_Gateway::supports_recurring()` returns false, so the
	# manual gateway is filtered out of the gateway picker. Likely fix
	# is `set_recurring(false)` in tests/e2e/cypress/fixtures/setup-product.php
	# or selecting a non-recurring plan in the spec. See tracking issue
	# for the full repro + verification plan.
	CHECKOUT_TESTS=(
	"tests/e2e/cypress/integration/020-free-trial-flow.spec.js"
	)

	TOTAL_FAILURES=0

	for TEST_SPEC in "${CHECKOUT_TESTS[@]}"; do
	echo "Running: $TEST_SPEC"

	npx cypress run \
	--config-file cypress.config.test.js \
	--spec "$TEST_SPEC" \
	--browser ${{ matrix.browser }}

	CYPRESS_EXIT_CODE=$?

	if [ $CYPRESS_EXIT_CODE -eq 0 ]; then
	echo "✅ $TEST_SPEC passed"
	else
	echo "❌ $TEST_SPEC failed with exit code $CYPRESS_EXIT_CODE"
	TOTAL_FAILURES=$((TOTAL_FAILURES + 1))
	fi
	done

	if [ $TOTAL_FAILURES -gt 0 ]; then
	echo "❌ $TOTAL_FAILURES checkout test(s) failed"
	exit 1
	fi

	echo "✅ All checkout tests passed!"

	- name: Run SSO Tests (After Setup)
	id: sso-tests
	run: \|
	set +e
	echo "=== Starting SSO Tests ==="

	SSO_TESTS=(
	"tests/e2e/cypress/integration/060-sso-cross-domain.spec.js"
	"tests/e2e/cypress/integration/065-sso-redirect-loop.spec.js"
	)

	TOTAL_FAILURES=0

	for TEST_SPEC in "${SSO_TESTS[@]}"; do
	echo "Running: $TEST_SPEC"

	npx cypress run \
	--config-file cypress.config.test.js \
	--spec "$TEST_SPEC" \
	--browser ${{ matrix.browser }}

	CYPRESS_EXIT_CODE=$?

	if [ $CYPRESS_EXIT_CODE -eq 0 ]; then
	echo "✅ $TEST_SPEC passed"
	else
	echo "❌ $TEST_SPEC failed with exit code $CYPRESS_EXIT_CODE"
	TOTAL_FAILURES=$((TOTAL_FAILURES + 1))
	fi
	done

	if [ $TOTAL_FAILURES -gt 0 ]; then
	echo "❌ $TOTAL_FAILURES SSO test(s) failed"
	exit 1
	fi

	echo "✅ All SSO tests passed!"

	- name: Run Stripe Tests (After Setup)
	id: stripe-tests
	env:
	STRIPE_TEST_PK_KEY: ${{ secrets.STRIPE_TEST_PK_KEY }}
	STRIPE_TEST_SK_KEY: ${{ secrets.STRIPE_TEST_SK_KEY }}
	run: \|
	if [ -z "$STRIPE_TEST_SK_KEY" ]; then
	echo "⏭️ Skipping Stripe tests: STRIPE_TEST_SK_KEY not available in this run."
	echo ""
	echo " The repo secrets STRIPE_TEST_PK_KEY and STRIPE_TEST_SK_KEY are configured,"
	echo " but GitHub Actions does NOT expose secrets to workflows triggered by"
	echo " pull_request events from forked repositories. This is a security feature,"
	echo " not a misconfiguration."
	echo ""
	echo " Stripe tests will run automatically when:"
	echo " • this branch is merged to main (push event), or"
	echo " • the PR is opened from a branch in the upstream Ultimate-Multisite repo."
	echo ""
	echo " To verify locally, run: gh secret list --repo Ultimate-Multisite/ultimate-multisite"
	exit 0
	fi

	set +e
	echo "=== Starting Stripe Tests ==="

	STRIPE_TESTS=(
	"tests/e2e/cypress/integration/030-stripe-checkout-flow.spec.js"
	"tests/e2e/cypress/integration/040-stripe-renewal-flow.spec.js"
	)

	TOTAL_FAILURES=0

	for TEST_SPEC in "${STRIPE_TESTS[@]}"; do
	echo "Running: $TEST_SPEC"

	npx cypress run \
	--config-file cypress.config.test.js \
	--spec "$TEST_SPEC" \
	--browser ${{ matrix.browser }}

	CYPRESS_EXIT_CODE=$?

	if [ $CYPRESS_EXIT_CODE -eq 0 ]; then
	echo "✅ $TEST_SPEC passed"
	else
	echo "❌ $TEST_SPEC failed with exit code $CYPRESS_EXIT_CODE"
	TOTAL_FAILURES=$((TOTAL_FAILURES + 1))
	fi
	done

	if [ $TOTAL_FAILURES -gt 0 ]; then
	echo "❌ $TOTAL_FAILURES Stripe test(s) failed"
	exit 1
	fi

	echo "✅ All Stripe tests passed!"

	- name: Run PayPal Tests (After Setup)
	id: paypal-tests
	env:
	PAYPAL_SANDBOX_CLIENT_ID: ${{ secrets.PAYPAL_SANDBOX_CLIENT_ID }}
	PAYPAL_SANDBOX_CLIENT_SECRET: ${{ secrets.PAYPAL_SANDBOX_CLIENT_SECRET }}
	run: \|
	if [ -z "$PAYPAL_SANDBOX_CLIENT_ID" ] \|\| [ -z "$PAYPAL_SANDBOX_CLIENT_SECRET" ]; then
	echo "⏭️ Skipping PayPal tests: PAYPAL_SANDBOX_CLIENT_ID or PAYPAL_SANDBOX_CLIENT_SECRET not available in this run."
	echo ""
	echo " The repo secrets PAYPAL_SANDBOX_CLIENT_ID and PAYPAL_SANDBOX_CLIENT_SECRET"
	echo " must be configured for these tests to run. GitHub Actions does NOT expose"
	echo " secrets to workflows triggered by pull_request events from forked"
	echo " repositories — this is a security feature, not a misconfiguration."
	echo ""
	echo " PayPal tests will run automatically when:"
	echo " • this branch is merged to main (push event), or"
	echo " • the PR is opened from a branch in the upstream Ultimate-Multisite repo"
	echo " AND the PAYPAL_SANDBOX_* secrets have been configured at the repo level."
	echo ""
	echo " To verify locally, run: gh secret list --repo Ultimate-Multisite/ultimate-multisite"
	exit 0
	fi

	set +e
	echo "=== Starting PayPal Tests ==="

	PAYPAL_TESTS=(
	"tests/e2e/cypress/integration/035-paypal-checkout-flow.spec.js"
	)

	TOTAL_FAILURES=0

	for TEST_SPEC in "${PAYPAL_TESTS[@]}"; do
	echo "Running: $TEST_SPEC"

	npx cypress run \
	--config-file cypress.config.test.js \
	--spec "$TEST_SPEC" \
	--browser ${{ matrix.browser }} \
	--env PAYPAL_SANDBOX_CLIENT_ID="$PAYPAL_SANDBOX_CLIENT_ID",PAYPAL_SANDBOX_CLIENT_SECRET="$PAYPAL_SANDBOX_CLIENT_SECRET"

	CYPRESS_EXIT_CODE=$?

	if [ $CYPRESS_EXIT_CODE -eq 0 ]; then
	echo "✅ $TEST_SPEC passed"
	else
	echo "❌ $TEST_SPEC failed with exit code $CYPRESS_EXIT_CODE"
	TOTAL_FAILURES=$((TOTAL_FAILURES + 1))
	fi
	done

	if [ $TOTAL_FAILURES -gt 0 ]; then
	echo "❌ $TOTAL_FAILURES PayPal test(s) failed"
	exit 1
	fi

	echo "✅ All PayPal tests passed!"

	- name: Fix permissions for Cypress output
	if: always()
	run: \|
	if [ -d tests/e2e/cypress ]; then
	sudo chown -R $USER:$USER tests/e2e/cypress
	else
	echo "Cypress output directory does not exist; skipping permission fix."
	fi

	- name: Upload Cypress screenshots
	if: always()
	continue-on-error: true
	uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
	with:
	name: cypress-screenshots-${{ matrix.php }}-${{ matrix.browser }}
	path: tests/e2e/cypress/screenshots

	- name: Upload Cypress videos
	if: always()
	continue-on-error: true
	uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
	with:
	name: cypress-videos-${{ matrix.php }}-${{ matrix.browser }}
	path: tests/e2e/cypress/videos

	- name: Stop WordPress Environment
	if: always()
	run: \|
	if [ -f package.json ]; then
	npm run env:stop
	else
	echo "package.json does not exist; checkout did not complete before cleanup."
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Robustness: null-deref, div-by-zero and control-flow guards (#1380) #2443

Workflow file

Robustness: null-deref, div-by-zero and control-flow guards (#1380) #2443

Uh oh!

Workflow file for this run