Skip to content

Fix stale workspace lock triage #655

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chubes4 merged 1 commit into from
Jun 12, 2026
Merged

Fix stale workspace lock triage #655

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 106 additions & 0 deletions inc/Cli/Commands/WorkspaceCommand.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -874,6 +874,7 @@ private function cleanup_run_control_job_ids( string $operation, int $job_id ):
}

private function render_cleanup_control_result( array $result, array $assoc_args ): void {
$result = $this->attach_current_workspace_lock_status($result);
$format = (string) ( $assoc_args['format'] ?? 'table' );
if ( 'json' === $format ) {
WP_CLI::log( (string) wp_json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
Expand All @@ -896,11 +897,37 @@ private function render_cleanup_control_result( array $result, array $assoc_args
if ( ! empty($result['remaining_work_summary']) && is_array($result['remaining_work_summary']) ) {
$this->render_cleanup_remaining_work_summary( (array) $result['remaining_work_summary']);
}
if ( ! empty($result['locks']['stale_locks']) && is_array($result['locks']['stale_locks']) ) {
$this->render_stale_lock_followup( (array) $result['locks']['stale_locks']);
}
if ( ! empty($result['evidence']) ) {
WP_CLI::log( (string) wp_json_encode($result['evidence'], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
}
}

/**
* Attach live workspace lock status to cleanup triage surfaces when available.
*
* @param array<string,mixed> $result Cleanup result.
* @return array<string,mixed>
*/
private function attach_current_workspace_lock_status( array $result ): array {
if ( isset($result['locks']) || ! class_exists(Workspace::class) || ! class_exists(WorkspaceMutationLock::class) ) {
return $result;
}

try {
$workspace = new Workspace();
$result['locks'] = WorkspaceMutationLock::status($workspace->get_path());
} catch ( \Throwable $e ) {
$result['locks'] = array(
'error' => $e->getMessage(),
);
}

return $result;
}

/**
* Render compact cleanup remaining-work summary.
*
Expand Down Expand Up @@ -3918,6 +3945,30 @@ static function ( array $lock ): array {
$this->format_items($items, array( 'lock_key', 'scope', 'state', 'age_seconds', 'safe_to_prune', 'owner_source', 'path' ), $assoc_args, 'lock_key');
}

$db_locks = (array) ( $db['locks'] ?? array() );
if ( ! empty($db_locks) ) {
$items = array_map(
static function ( array $lock ): array {
return array(
'lock_key' => (string) ( $lock['lock_key'] ?? '' ),
'scope' => (string) ( $lock['scope'] ?? '' ),
'state' => (string) ( $lock['state'] ?? $lock['status'] ?? '' ),
'owner' => (string) ( $lock['owner'] ?? '' ),
'age_seconds' => $lock['age_seconds'] ?? null,
'expires_at' => (string) ( $lock['expires_at'] ?? '' ),
);
},
$db_locks
);
WP_CLI::log('');
WP_CLI::log('Database lock rows:');
$this->format_items($items, array( 'lock_key', 'scope', 'state', 'owner', 'age_seconds', 'expires_at' ), $assoc_args, 'lock_key');
}

if ( ! empty($status['stale_locks']) && is_array($status['stale_locks']) ) {
$this->render_stale_lock_followup( (array) $status['stale_locks']);
}

$guidance = (array) ( $fs['guidance'] ?? $status['recovery_guidance'] ?? array() );
if ( ! empty($guidance) ) {
WP_CLI::log(sprintf('Status: %s', (string) ( $guidance['status_command'] ?? 'wp datamachine-code workspace worktree locks --format=json' )));
Expand All @@ -3926,6 +3977,57 @@ static function ( array $lock ): array {
}
}

/**
* Render stale lock follow-up rows with exact preview/apply commands.
*
* @param array<string,mixed> $report Stale lock report.
*/
private function render_stale_lock_followup( array $report ): void {
if ( (int) ( $report['count'] ?? 0 ) <= 0 ) {
return;
}

WP_CLI::log('');
WP_CLI::log('Stale workspace locks:');
WP_CLI::log(sprintf('Preview: %s', (string) ( $report['preview_command'] ?? 'wp datamachine-code workspace worktree locks --prune-stale --dry-run --format=json' )));
WP_CLI::log(sprintf('Apply: %s', (string) ( $report['apply_command'] ?? 'wp datamachine-code workspace worktree locks --prune-stale --format=json' )));
WP_CLI::log((string) ( $report['safety'] ?? 'Active filesystem flocks are reported and protected.' ));

$rows = array();
foreach ( (array) ( $report['database'] ?? array() ) as $row ) {
if ( ! is_array($row) ) {
continue;
}
$rows[] = array(
'source' => 'database',
'lock_key' => (string) ( $row['lock_key'] ?? '' ),
'scope' => (string) ( $row['scope'] ?? '' ),
'owner' => (string) ( $row['owner'] ?? '' ),
'session' => (string) ( $row['session'] ?? '' ),
'age_seconds' => $row['age_seconds'] ?? null,
'live_flock_present' => ! empty($row['live_flock_present']) ? 'yes' : 'no',
'safe_to_prune' => ! empty($row['safe_to_prune']) ? 'yes' : 'no',
);
}
foreach ( (array) ( $report['filesystem'] ?? array() ) as $row ) {
if ( ! is_array($row) ) {
continue;
}
$rows[] = array(
'source' => 'filesystem',
'lock_key' => (string) ( $row['lock_key'] ?? '' ),
'scope' => (string) ( $row['scope'] ?? '' ),
'owner' => '',
'session' => '',
'age_seconds' => $row['age_seconds'] ?? null,
'live_flock_present' => ! empty($row['live_flock_present']) ? 'yes' : 'no',
'safe_to_prune' => ! empty($row['safe_to_prune']) ? 'yes' : 'no',
);
}

$this->format_items($rows, array( 'source', 'lock_key', 'scope', 'owner', 'session', 'age_seconds', 'live_flock_present', 'safe_to_prune' ), array( 'format' => 'table' ), 'lock_key');
}

private function render_workspace_error( \WP_Error $error ): void {
$data = (array) $error->get_error_data();
if ( 'workspace_repo_busy' !== $error->get_error_code() && ! empty($data['next_commands']) && is_array($data['next_commands']) ) {
Expand Down Expand Up @@ -4165,6 +4267,10 @@ private function render_workspace_hygiene_report( array $report, array $assoc_ar
'metric'
);

if ( ! empty($locks['stale_locks']) && is_array($locks['stale_locks']) ) {
$this->render_stale_lock_followup( (array) $locks['stale_locks']);
}

$duplicates = (array) ( $worktrees['duplicates'] ?? array() );
if ( array() !== $duplicates ) {
WP_CLI::log('');
Expand Down
4 changes: 2 additions & 2 deletions inc/Workspace/WorkspaceHygieneReport.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ private function build_workspace_inventory_rows(): array {

$rows = array();
foreach ( $entries as $entry ) {
if ( '.' === $entry || '..' === $entry ) {
if ( '.' === $entry || '..' === $entry || str_starts_with((string) $entry, '.') ) {
continue;
}

Expand Down Expand Up @@ -444,7 +444,7 @@ private function build_workspace_size_report( int $limit ): array {
$dirs = array_values(
array_filter(
$entries,
fn( $entry ) => '.' !== $entry && '..' !== $entry && is_dir($this->workspace_path . '/' . $entry)
fn( $entry ) => '.' !== $entry && '..' !== $entry && ! str_starts_with((string) $entry, '.') && is_dir($this->workspace_path . '/' . $entry)
)
);
sort($dirs, SORT_NATURAL);
Expand Down
84 changes: 76 additions & 8 deletions inc/Workspace/WorkspaceLockStore.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ public static function status(): array {
'active_keys' => self::lock_keys_for_status('active', false),
'stale' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM {$table} WHERE status = %s AND expires_at < %s", 'active', $now)),
'stale_keys' => self::lock_keys_for_status('active', true),
'locks' => array_merge(self::lock_rows_for_status('active', false), self::lock_rows_for_status('active', true)),
'released' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM {$table} WHERE status = %s", 'released')),
'total' => (int) $wpdb->get_var("SELECT COUNT(*) FROM {$table}"),
);
Expand Down Expand Up @@ -211,7 +212,7 @@ public static function active_lock( string $lock_key, string $scope ): array|nul
*
* @return array<string,mixed>
*/
public static function prune_expired(): array {
public static function prune_expired( array $protected_lock_keys = array() ): array {
$status = self::status();
if ( empty($status['available']) ) {
return array(
Expand All @@ -224,22 +225,35 @@ public static function prune_expired(): array {
}

global $wpdb;
$table = self::table_name();
$now = gmdate('Y-m-d H:i:s');
$released_cutoff = gmdate('Y-m-d H:i:s', time() - self::released_ttl_seconds());
$table = self::table_name();
$now = gmdate('Y-m-d H:i:s');
$released_cutoff = gmdate('Y-m-d H:i:s', time() - self::released_ttl_seconds());
$protected_lock_keys = array_values(array_unique(array_filter(array_map('strval', $protected_lock_keys))));
$protected_sql = '';
$protected_args = array();
if ( array() !== $protected_lock_keys ) {
$protected_sql = ' AND lock_key NOT IN (' . implode(', ', array_fill(0, count($protected_lock_keys), '%s')) . ')';
$protected_args = $protected_lock_keys;
}

// phpcs:disable WordPress.DB.PreparedSQL -- Table name from $wpdb->prefix, not user input.
$wpdb->query($wpdb->prepare("UPDATE {$table} SET status = %s WHERE status = %s AND expires_at < %s", 'stale', 'active', $now));
// phpcs:disable WordPress.DB.PreparedSQL -- Table name from $wpdb->prefix, not user input.
$mark_sql = "UPDATE {$table} SET status = %s WHERE status = %s AND expires_at < %s{$protected_sql}";
$mark_args = array_merge(array( 'stale', 'active', $now ), $protected_args);
$wpdb->query(call_user_func_array(array( $wpdb, 'prepare' ), array_merge(array( $mark_sql ), $mark_args)));
$marked = (int) $wpdb->rows_affected;

$wpdb->query($wpdb->prepare("DELETE FROM {$table} WHERE status IN (%s, %s) AND COALESCE(released_at, expires_at) < %s", 'released', 'stale', $released_cutoff));
// phpcs:enable WordPress.DB.PreparedSQL
$delete_sql = "DELETE FROM {$table} WHERE status IN (%s, %s) AND COALESCE(released_at, expires_at) < %s{$protected_sql}";
$delete_args = array_merge(array( 'released', 'stale', $released_cutoff ), $protected_args);
$wpdb->query(call_user_func_array(array( $wpdb, 'prepare' ), array_merge(array( $delete_sql ), $delete_args)));
// phpcs:enable WordPress.DB.PreparedSQL
$deleted = (int) $wpdb->rows_affected;

return array(
'available' => true,
'active_marked_stale' => $marked,
'released_deleted' => $deleted,
'protected_active' => count($protected_lock_keys),
'protected_keys' => $protected_lock_keys,
'before' => $status,
'after' => self::status(),
);
Expand Down Expand Up @@ -322,6 +336,60 @@ private static function lock_keys_for_status( string $status, bool $expired ): a
);
}

/**
* Return lock rows for owner/session/age diagnostics.
*
* @return array<int,array<string,mixed>>
*/
private static function lock_rows_for_status( string $status, bool $expired ): array {
global $wpdb;
if ( ! is_object($wpdb) || ! is_callable(array( $wpdb, 'prepare' )) || ! is_callable(array( $wpdb, 'get_results' )) ) {
return array();
}

$table = self::table_name();
$now = gmdate('Y-m-d H:i:s');
$operator = $expired ? '<' : '>=';

// phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- Table name from $wpdb->prefix, operator is constant-selected above.
$query = call_user_func(array( $wpdb, 'prepare' ), "SELECT id, lock_key, purpose, scope, owner, run_id, job_id, status, acquired_at, heartbeat_at, expires_at, released_at, metadata_json FROM {$table} WHERE status = %s AND expires_at {$operator} %s ORDER BY acquired_at DESC, id DESC LIMIT 25", $status, $now);
$rows = call_user_func(array( $wpdb, 'get_results' ), $query, ARRAY_A);
if ( ! is_array($rows) ) {
return array();
}

return array_values(array_map(static fn( array $row ): array => self::normalize_lock_row($row, $expired ? 'stale' : 'active'), $rows));
}

/**
* @param array<string,mixed> $row Raw DB row.
* @return array<string,mixed>
*/
private static function normalize_lock_row( array $row, string $state ): array {
$row['id'] = (int) ( $row['id'] ?? 0 );
$row['job_id'] = isset($row['job_id']) ? (int) $row['job_id'] : null;
$row['state'] = $state;
$row['metadata'] = self::decode_metadata( (string) ( $row['metadata_json'] ?? '' ) );
unset($row['metadata_json']);

$acquired = self::timestamp_seconds( (string) ( $row['acquired_at'] ?? '' ) );
$heartbeat = self::timestamp_seconds( (string) ( $row['heartbeat_at'] ?? '' ) );
$expires = self::timestamp_seconds( (string) ( $row['expires_at'] ?? '' ) );
$time = time();
if ( null !== $acquired ) {
$row['age_seconds'] = max(0, $time - $acquired);
}
if ( null !== $heartbeat ) {
$row['heartbeat_age_seconds'] = max(0, $time - $heartbeat);
}
if ( null !== $expires ) {
$row['expires_age_seconds'] = 'stale' === $state ? max(0, $time - $expires) : 0;
$row['expires_in_seconds'] = 'active' === $state ? max(0, $expires - $time) : 0;
}

return $row;
}

private static function expires_seconds(): int {
$seconds = self::DEFAULT_EXPIRES_SECONDS;
if ( function_exists('apply_filters') ) {
Expand Down
Loading
Loading