From 2ee7cbdbd10eb8c98a59001f274e4c85327f21dd Mon Sep 17 00:00:00 2001 From: Vinod Chitrali Date: Fri, 17 Jul 2026 18:28:56 +0000 Subject: [PATCH] feat: route compute-tray power control through machine state controller Add machine maintenance power operations mirroring the existing switch and power-shelf state-controller maintenance pattern. When compute_tray_use_state_controller is enabled, ComponentPowerControl for machines queues a maintenance request instead of issuing Redfish directly. Model and persistence: - Introduce MachineMaintenanceOperation (PowerOn, PowerOff, Reset) and MachineMaintenanceRequest. - Add ManagedHostState::Maintenance to execute queued operations. - Add machines.machine_maintenance_requested JSONB column and DB helpers to set/clear the pending request. Machine state handler: - New maintenance handler dispatches power actions via compute_tray backend. - Ready and Failed states transition into Maintenance when a request is pending; success returns to Ready, failure clears the request and moves to Failed to avoid retry loops. - Wire component_manager and credential_manager into machine controller services for endpoint/credential resolution. Component manager and API: - Implement request_machine_maintenance_via_state_controller. - Replace the unimplemented compute-tray state-controller path in component_power_control with the maintenance queue. - Map PowerAction to MachineMaintenanceOperation consistently with switches. Signed-off-by: Vinod Chitrali --- Cargo.lock | 1 + .../src/handlers/component_manager.rs | 92 +++- crates/api-core/src/setup.rs | 2 + .../src/tests/common/api_fixtures/mod.rs | 5 + ...17120000_machine_maintenance_requested.sql | 7 + crates/api-db/src/machine.rs | 39 +- crates/api-model/src/machine/json.rs | 5 +- crates/api-model/src/machine/mod.rs | 43 ++ crates/api-model/src/machine/slas.rs | 2 + .../src/test_support/machine_snapshot.rs | 1 + .../src/component_manager.rs | 93 ++++ crates/machine-controller/Cargo.toml | 2 + crates/machine-controller/src/context.rs | 5 + crates/machine-controller/src/handler.rs | 19 + .../src/handler/maintenance.rs | 309 ++++++++++++ crates/machine-controller/src/io.rs | 12 +- .../tests/integration/env.rs | 24 + .../tests/integration/main.rs | 1 + .../tests/integration/maintenance.rs | 456 ++++++++++++++++++ 19 files changed, 1108 insertions(+), 10 deletions(-) create mode 100644 crates/api-db/migrations/20260717120000_machine_maintenance_requested.sql create mode 100644 crates/machine-controller/src/handler/maintenance.rs create mode 100644 crates/machine-controller/tests/integration/maintenance.rs diff --git a/Cargo.lock b/Cargo.lock index 9d370b92bb..71e4e49e80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2391,6 +2391,7 @@ dependencies = [ "carbide-utils", "carbide-uuid", "chrono", + "component-manager", "config-version", "duration-str", "eyre", diff --git a/crates/api-core/src/handlers/component_manager.rs b/crates/api-core/src/handlers/component_manager.rs index 22db961115..6576e173e2 100644 --- a/crates/api-core/src/handlers/component_manager.rs +++ b/crates/api-core/src/handlers/component_manager.rs @@ -43,8 +43,8 @@ use model::component_manager::{ PowerShelfComponent, }; use model::firmware::FirmwareComponentType; -use model::machine::Machine; use model::machine::machine_search_config::MachineSearchConfig; +use model::machine::{Machine, MachineMaintenanceOperation}; use model::power_shelf::PowerShelfMaintenanceOperation; use model::rack::{FirmwareUpgradeJob, MaintenanceActivity}; use model::switch::SwitchMaintenanceOperation; @@ -307,6 +307,18 @@ fn map_switch_maintenance_operation(action: PowerAction) -> SwitchMaintenanceOpe } } +fn map_machine_maintenance_operation(action: PowerAction) -> MachineMaintenanceOperation { + match action { + PowerAction::On => MachineMaintenanceOperation::PowerOn, + PowerAction::GracefulShutdown | PowerAction::ForceOff => { + MachineMaintenanceOperation::PowerOff + } + PowerAction::GracefulRestart | PowerAction::ForceRestart | PowerAction::AcPowercycle => { + MachineMaintenanceOperation::Reset + } + } +} + fn map_power_shelf_maintenance_operation( action: PowerAction, ) -> Result { @@ -362,6 +374,47 @@ fn switch_maintenance_request_result_to_component_result( } } +async fn queue_machine_power_control_via_state_controller( + api: &Api, + cm: &ComponentManager, + machine_ids: &[carbide_uuid::machine::MachineId], + action: PowerAction, +) -> Result, Status> { + let operation = map_machine_maintenance_operation(action); + queue_machine_maintenance_via_state_controller(api, cm, machine_ids, operation).await +} + +async fn queue_machine_maintenance_via_state_controller( + api: &Api, + cm: &ComponentManager, + machine_ids: &[carbide_uuid::machine::MachineId], + operation: MachineMaintenanceOperation, +) -> Result, Status> { + let results = cm + .request_machine_maintenance_via_state_controller( + &api.database_connection, + machine_ids, + operation, + "component-manager", + ) + .await + .map_err(component_manager_error_to_status)?; + + Ok(results + .iter() + .map(machine_maintenance_request_result_to_component_result) + .collect()) +} + +fn machine_maintenance_request_result_to_component_result( + result: &component_manager::component_manager::MachineMaintenanceRequestResult, +) -> rpc::ComponentResult { + match &result.error { + Some(error) => error_result(&result.machine_id.to_string(), error.clone()), + None => success_result(&result.machine_id.to_string()), + } +} + async fn queue_power_shelf_power_control_via_state_controller( api: &Api, power_shelf_ids: &[PowerShelfId], @@ -1580,10 +1633,15 @@ pub(crate) async fn component_power_control( } rpc::component_power_control_request::Target::MachineIds(list) => { if cm.compute_tray_use_state_controller && !bypass_state_controller { - // TODO: implement state controller path for compute tray power control - return Err(Status::unimplemented( - "compute tray power control through the state controller is not yet supported", - )); + let results = queue_machine_power_control_via_state_controller( + api, + cm, + &list.machine_ids, + action, + ) + .await?; + let ips = Vec::new(); + (results, ips) } else { let resolved = resolve_compute_tray_endpoints(api, &list.machine_ids).await?; @@ -3305,6 +3363,30 @@ mod tests { ); } + #[test] + fn map_machine_maintenance_operation_variants() { + use model::machine::MachineMaintenanceOperation; + + use super::map_machine_maintenance_operation; + + assert_eq!( + map_machine_maintenance_operation(PowerAction::On), + MachineMaintenanceOperation::PowerOn, + ); + assert_eq!( + map_machine_maintenance_operation(PowerAction::ForceOff), + MachineMaintenanceOperation::PowerOff, + ); + assert_eq!( + map_machine_maintenance_operation(PowerAction::GracefulShutdown), + MachineMaintenanceOperation::PowerOff, + ); + assert_eq!( + map_machine_maintenance_operation(PowerAction::ForceRestart), + MachineMaintenanceOperation::Reset, + ); + } + #[test] fn map_power_shelf_maintenance_operation_variants() { use model::power_shelf::PowerShelfMaintenanceOperation; diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index 0573e97766..ea6daea4ae 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -1362,6 +1362,8 @@ async fn initialize_and_start_controllers<'a>( redfish_client_pool: shared_redfish_pool.clone(), ipmi_tool: ipmi_tool.clone(), site_config: carbide_config.machine_state_handler_site_config().into(), + component_manager: component_manager.clone().map(Arc::new), + credential_manager: credential_manager.clone(), per_object_metrics_registry: per_object_metrics_registry.clone(), } .into(), diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 703543cf05..7191b0a487 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -336,6 +336,8 @@ impl TestEnv { redfish_client_pool: self.redfish_sim.clone(), ipmi_tool: self.ipmi_tool.clone(), site_config: self.config.machine_state_handler_site_config().into(), + component_manager: self.test_component_manager.clone(), + credential_manager: self.test_credential_manager.clone(), per_object_metrics_registry: self.per_object_metrics_registry(), } } @@ -420,6 +422,7 @@ impl TestEnv { ManagedHostState::HostInit { machine_state: mc } } ManagedHostState::Ready => state.clone(), + ManagedHostState::Maintenance { .. } => state.clone(), ManagedHostState::Assigned { .. } => state.clone(), ManagedHostState::WaitingForCleanup { .. } => state.clone(), ManagedHostState::Created => state.clone(), @@ -1542,6 +1545,8 @@ pub async fn create_test_env_with_overrides( redfish_client_pool: redfish_sim.clone(), ipmi_tool: ipmi_tool.clone(), site_config: config.machine_state_handler_site_config().into(), + component_manager: test_component_manager.clone(), + credential_manager: credential_manager.clone(), per_object_metrics_registry: per_object_metrics_registry.clone(), } .into(), diff --git a/crates/api-db/migrations/20260717120000_machine_maintenance_requested.sql b/crates/api-db/migrations/20260717120000_machine_maintenance_requested.sql new file mode 100644 index 0000000000..6373d35c3f --- /dev/null +++ b/crates/api-db/migrations/20260717120000_machine_maintenance_requested.sql @@ -0,0 +1,7 @@ +-- Add machine_maintenance_requested column to machines table. +-- machine_maintenance_requested: when set by an external entity, the state controller +-- transitions the host from Ready (or Failed) into Maintenance to execute the requested +-- operation (PowerOn / PowerOff / Reset). Mirrors switches.switch_maintenance_requested. + +ALTER TABLE machines + ADD COLUMN machine_maintenance_requested JSONB; diff --git a/crates/api-db/src/machine.rs b/crates/api-db/src/machine.rs index 6f50523c6d..6a90e6726e 100644 --- a/crates/api-db/src/machine.rs +++ b/crates/api-db/src/machine.rs @@ -50,8 +50,8 @@ use model::machine::upgrade_policy::AgentUpgradePolicy; use model::machine::{ Dpf, DpuInfo, DpuInfoStatusObservation, DpuOsOperationalState, DpuRepresentorStatus, FailureDetails, HostProfile, Machine, MachineInterfaceSnapshot, MachineLastRebootRequested, - MachineLastRebootRequestedMode, MachineValidationContext, ManagedHostState, ReprovisionRequest, - UpgradeDecision, + MachineLastRebootRequestedMode, MachineMaintenanceOperation, MachineValidationContext, + ManagedHostState, ReprovisionRequest, UpgradeDecision, }; use model::machine_interface_address::MachineInterfaceAssociation; use model::metadata::Metadata; @@ -2448,6 +2448,41 @@ pub async fn set_machine_validation_request( Ok(()) } +pub async fn set_machine_maintenance_requested( + txn: &mut PgConnection, + machine_id: MachineId, + initiator: &str, + operation: MachineMaintenanceOperation, +) -> DatabaseResult<()> { + let req = model::machine::MachineMaintenanceRequest { + requested_at: Utc::now(), + initiator: initiator.to_string(), + operation, + }; + let query = "UPDATE machines SET machine_maintenance_requested = $1 WHERE id = $2 RETURNING id"; + sqlx::query_as::<_, MachineId>(query) + .bind(sqlx::types::Json(req)) + .bind(machine_id) + .fetch_one(txn) + .await + .map_err(|e| DatabaseError::new("set_machine_maintenance_requested", e))?; + Ok(()) +} + +pub async fn clear_machine_maintenance_requested( + txn: &mut PgConnection, + machine_id: MachineId, +) -> DatabaseResult<()> { + let query = + "UPDATE machines SET machine_maintenance_requested = NULL WHERE id = $1 RETURNING id"; + sqlx::query_as::<_, MachineId>(query) + .bind(machine_id) + .fetch_one(txn) + .await + .map_err(|e| DatabaseError::new("clear_machine_maintenance_requested", e))?; + Ok(()) +} + pub async fn update_dpu_asns( db_pool: &Pool, common_pools: &CommonPools, diff --git a/crates/api-model/src/machine/json.rs b/crates/api-model/src/machine/json.rs index 56bf193210..0f32f02794 100644 --- a/crates/api-model/src/machine/json.rs +++ b/crates/api-model/src/machine/json.rs @@ -36,7 +36,8 @@ use crate::machine::spx::MachineSpxStatusObservation; use crate::machine::topology::MachineTopology; use crate::machine::{ Dpf, FailureDetails, HostProfile, HostReprovisionRequest, Machine, MachineInterfaceSnapshot, - MachineLastRebootRequested, ManagedHostState, ReprovisionRequest, UpgradeDecision, + MachineLastRebootRequested, MachineMaintenanceRequest, ManagedHostState, ReprovisionRequest, + UpgradeDecision, }; use crate::metadata::Metadata; use crate::power_manager::PowerOptions; @@ -73,6 +74,7 @@ pub struct MachineSnapshotPgJson { pub failure_details: FailureDetails, pub reprovisioning_requested: Option, pub host_reprovisioning_requested: Option, + pub machine_maintenance_requested: Option, pub manual_firmware_upgrade_completed: Option>, pub bios_password_set_time: Option>, pub last_machine_validation_time: Option>, @@ -189,6 +191,7 @@ impl TryFrom for Machine { failure_details: value.failure_details, reprovision_requested: value.reprovisioning_requested, host_reprovision_requested: value.host_reprovisioning_requested, + machine_maintenance_requested: value.machine_maintenance_requested, manual_firmware_upgrade_completed: value.manual_firmware_upgrade_completed, dpu_agent_upgrade_requested: value.dpu_agent_upgrade_requested, health_reports: value.health_reports.unwrap_or_default(), diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index 537cabffda..41990dbe0a 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -817,6 +817,10 @@ pub struct Machine { /// Last time when host reprovision requested pub host_reprovision_requested: Option, + /// When set by an external entity, the state controller transitions the host into + /// [`ManagedHostState::Maintenance`] to execute the requested operation. + pub machine_maintenance_requested: Option, + /// Does the forge-dpu-agent on this DPU need upgrading? pub dpu_agent_upgrade_requested: Option, @@ -1206,6 +1210,12 @@ pub enum ManagedHostState { }, /// Host is Ready for instance creation. Ready, + + /// Host is executing an operator-requested maintenance operation. + Maintenance { + operation: MachineMaintenanceOperation, + }, + /// Host is assigned to an Instance. Assigned { instance_state: InstanceState, @@ -1357,6 +1367,11 @@ impl std::fmt::Display for ValidationState { pub const MAX_FIRMWARE_UPGRADE_RETRIES: u32 = 5; impl ManagedHostState { + /// Builds the controller state for a requested maintenance operation. + pub fn maintenance_for_operation(operation: MachineMaintenanceOperation) -> Self { + Self::Maintenance { operation } + } + pub fn as_reprovision_state(&self, dpu_id: &MachineId) -> Option<&ReprovisionState> { match self { ManagedHostState::DPUReprovision { dpu_states } => dpu_states.states.get(dpu_id), @@ -2175,6 +2190,25 @@ pub struct ReprovisionRequest { pub restart_reprovision_requested_at: DateTime, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "operation", rename_all = "lowercase")] +#[allow(clippy::enum_variant_names)] +pub enum MachineMaintenanceOperation { + /// Power on the host. + PowerOn, + /// Power off the host. + PowerOff, + /// Reset the host (restart / AC power cycle). + Reset, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MachineMaintenanceRequest { + pub requested_at: DateTime, + pub initiator: String, + pub operation: MachineMaintenanceOperation, +} + /// Struct to store information if host reprovision is requested. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HostReprovisionRequest { @@ -2321,6 +2355,9 @@ impl Display for ManagedHostState { write!(f, "HostInitializing/{machine_state}") } ManagedHostState::Ready => write!(f, "Ready"), + ManagedHostState::Maintenance { operation } => { + write!(f, "Maintenance({operation:?})") + } ManagedHostState::Assigned { instance_state, .. } => match instance_state { InstanceState::DPUReprovision { dpu_states } => { let dpu_lowest_state = dpu_states @@ -2413,6 +2450,9 @@ impl ManagedHostState { format!("HostInitializing/{machine_state}") } ManagedHostState::Ready => "Ready".to_string(), + ManagedHostState::Maintenance { operation } => { + format!("Maintenance({operation:?})") + } ManagedHostState::Assigned { instance_state } => match instance_state { InstanceState::DPUReprovision { dpu_states } => { format!( @@ -2612,6 +2652,9 @@ pub fn state_sla( _ => StateSla::with_sla(slas::HOST_INIT, time_in_state), }, ManagedHostState::Ready => StateSla::no_sla(), + ManagedHostState::Maintenance { .. } => { + StateSla::with_sla(slas::MAINTENANCE, time_in_state) + } ManagedHostState::Assigned { instance_state } => match instance_state { InstanceState::Ready => StateSla::no_sla(), InstanceState::BootingWithDiscoveryImage { retry } => { diff --git a/crates/api-model/src/machine/slas.rs b/crates/api-model/src/machine/slas.rs index c10803da72..45c8edd18e 100644 --- a/crates/api-model/src/machine/slas.rs +++ b/crates/api-model/src/machine/slas.rs @@ -57,6 +57,8 @@ pub const ASSIGNED: Duration = Duration::from_secs(30 * 60); pub const ASSIGNED_HOST_PLATFORM_CONFIGURATION: Duration = Duration::from_secs(90 * 60); pub const VALIDATION: Duration = Duration::from_secs(30 * 60); +pub const MAINTENANCE: Duration = Duration::from_secs(5 * 60); + /// Configuration for machine state SLA durations. #[derive(Clone, Debug, PartialEq)] pub struct MachineSlaConfig { diff --git a/crates/api-model/src/test_support/machine_snapshot.rs b/crates/api-model/src/test_support/machine_snapshot.rs index 5ef509bb81..cd2ff42c3b 100644 --- a/crates/api-model/src/test_support/machine_snapshot.rs +++ b/crates/api-model/src/test_support/machine_snapshot.rs @@ -359,6 +359,7 @@ pub fn machine_snapshot_pg_json(machine_id: MachineId) -> MachineSnapshotPgJson }; MachineSnapshotPgJson { + machine_maintenance_requested: None, id: machine_id, rack_id: Some("rack-bench-01".parse().expect("valid rack id")), created: fixture_time(0), diff --git a/crates/component-manager/src/component_manager.rs b/crates/component-manager/src/component_manager.rs index dc1cc3e8af..fa67b37f6a 100644 --- a/crates/component-manager/src/component_manager.rs +++ b/crates/component-manager/src/component_manager.rs @@ -7,10 +7,12 @@ use std::sync::Arc; use carbide_rack::firmware_object::rack_maintenance_access_token_key; use carbide_redfish::libredfish::RedfishClientPool; use carbide_secrets::credentials::{CredentialManager, Credentials}; +use carbide_uuid::machine::MachineId; use carbide_uuid::rack::RackId; use carbide_uuid::switch::SwitchId; use db::{ObjectColumnFilter, WithTransaction}; use librms::RmsApi; +use model::machine::MachineMaintenanceOperation; use model::rack::{MaintenanceActivity, MaintenanceScope, RackState}; use model::rack_type::RackProfileConfig; use model::switch::SwitchMaintenanceOperation; @@ -55,6 +57,12 @@ pub struct SwitchMaintenanceRequestResult { pub error: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MachineMaintenanceRequestResult { + pub machine_id: MachineId, + pub error: Option, +} + /// Which rack states a maintenance caller is willing to schedule from. /// /// Automatic maintenance uses [`RequireReady`](Self::RequireReady), while the @@ -352,6 +360,91 @@ impl ComponentManager { .map_err(|error| ComponentManagerError::Internal(error.to_string()))? } + pub async fn request_machine_maintenance_via_state_controller( + &self, + db_pool: &PgPool, + machine_ids: &[MachineId], + operation: MachineMaintenanceOperation, + initiator: &str, + ) -> Result, ComponentManagerError> { + if !self.compute_tray_use_state_controller { + return Err(ComponentManagerError::InvalidArgument( + "compute_tray_use_state_controller is disabled; machine maintenance through the state controller is unavailable" + .to_string(), + )); + } + + let machine_ids = machine_ids.to_vec(); + let initiator = initiator.to_string(); + db_pool + .with_txn(|txn| { + Box::pin(async move { + let existing = db::machine::find( + txn.as_mut(), + db::ObjectFilter::List(&machine_ids), + model::machine::machine_search_config::MachineSearchConfig::default(), + ) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))?; + + let by_id: HashMap = existing + .into_iter() + .map(|machine| (machine.id, machine)) + .collect(); + let mut results = Vec::with_capacity(machine_ids.len()); + + for machine_id in &machine_ids { + let Some(machine) = by_id.get(machine_id) else { + results.push(MachineMaintenanceRequestResult { + machine_id: *machine_id, + error: Some(format!("machine {machine_id} not found")), + }); + continue; + }; + + if !machine_id.machine_type().is_host() { + results.push(MachineMaintenanceRequestResult { + machine_id: *machine_id, + error: Some(format!("machine {machine_id} is not a host machine")), + }); + continue; + } + + if matches!( + machine.state.value, + model::machine::ManagedHostState::ForceDeletion + ) { + results.push(MachineMaintenanceRequestResult { + machine_id: *machine_id, + error: Some(format!( + "machine {machine_id} is marked for forced deletion" + )), + }); + continue; + } + + db::machine::set_machine_maintenance_requested( + txn, + *machine_id, + &initiator, + operation, + ) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))?; + + results.push(MachineMaintenanceRequestResult { + machine_id: *machine_id, + error: None, + }); + } + + Ok(results) + }) + }) + .await + .map_err(|error| ComponentManagerError::Internal(error.to_string()))? + } + pub async fn configure_switch_certificate( &self, endpoint: &SwitchEndpoint, diff --git a/crates/machine-controller/Cargo.toml b/crates/machine-controller/Cargo.toml index 2aba63d481..cbbb435be8 100644 --- a/crates/machine-controller/Cargo.toml +++ b/crates/machine-controller/Cargo.toml @@ -45,6 +45,7 @@ carbide-redfish = { path = "../redfish", default-features = false } carbide-secrets = { path = "../secrets" } carbide-state-controller-common = { path = "../state-controller-common", default-features = false } carbide-uuid = { path = "../uuid", default-features = false } +component-manager = { path = "../component-manager" } config-version = { path = "../config-version", default-features = false } state-controller = { path = "../state-controller" } @@ -76,6 +77,7 @@ version-compare = { workspace = true } [dev-dependencies] carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-redfish = { path = "../redfish", features = ["test-support"] } +carbide-secrets = { path = "../secrets", features = ["test-support"] } carbide-test-harness = { path = "../test-harness" } carbide-test-support = { path = "../test-support" } diff --git a/crates/machine-controller/src/context.rs b/crates/machine-controller/src/context.rs index d6bca9ffd2..b433182535 100644 --- a/crates/machine-controller/src/context.rs +++ b/crates/machine-controller/src/context.rs @@ -20,6 +20,8 @@ use std::sync::Arc; use carbide_health_metrics::PerObjectMetricsRegistry; use carbide_ipmi::IPMITool; use carbide_redfish::libredfish::RedfishClientPool; +use carbide_secrets::credentials::CredentialManager; +use component_manager::component_manager::ComponentManager; use db::db_read::PgPoolReader; use libredfish::Redfish; use model::machine::Machine; @@ -48,6 +50,9 @@ pub struct MachineStateHandlerServices { pub ipmi_tool: Arc, /// Configuration used by MachineStateHandler. pub site_config: Arc, + /// Optional Component Manager backend for rack-scale maintenance operations. + pub component_manager: Option>, + pub credential_manager: Arc, /// Shared registry backing the generic per-object health metrics. pub per_object_metrics_registry: Arc, } diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 19bd7d1345..6cca7b849c 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -121,6 +121,7 @@ mod firmware_artifact; mod helpers; mod host_boot_config; mod machine_validation; +mod maintenance; mod power; mod sku; #[cfg(test)] @@ -830,6 +831,11 @@ impl MachineStateHandler { return Ok(outcome); } + if let Some(outcome) = maintenance::maintenance_transition_if_requested(mh_snapshot) + { + return Ok(outcome); + } + // Check if instance to be created. if mh_snapshot.instance.is_some() { return Ok(StateHandlerOutcome::transition( @@ -1297,6 +1303,15 @@ impl MachineStateHandler { machine_id, retry_count, } => { + if let Some(outcome) = maintenance::maintenance_transition_if_requested(mh_snapshot) + { + // Clear stale host failure details before accepting maintenance, + // matching other Failed recovery branches that exit with a txn. + let mut txn = ctx.services.db_pool.begin().await?; + db::machine::clear_failure_details(host_machine_id, &mut txn).await?; + return Ok(outcome.with_txn(txn)); + } + match details.cause { // DPU discovery failed needs more logic to handle. // DPU discovery can failed from multiple states init, @@ -1544,6 +1559,10 @@ impl MachineStateHandler { .await } + ManagedHostState::Maintenance { .. } => { + maintenance::handle_maintenance(host_machine_id, mh_snapshot, ctx).await + } + // ManagedHostState::Measuring is introduced into the flow when // attestation_enabled is set to true (defaults to false), and // is triggered when a machine being in Ready state suddently diff --git a/crates/machine-controller/src/handler/maintenance.rs b/crates/machine-controller/src/handler/maintenance.rs new file mode 100644 index 0000000000..6c87e149bb --- /dev/null +++ b/crates/machine-controller/src/handler/maintenance.rs @@ -0,0 +1,309 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Handler for [`ManagedHostState::Maintenance`]. + +use carbide_secrets::credentials::{ + BmcCredentialType, CredentialKey, CredentialManager, Credentials, +}; +use carbide_uuid::machine::MachineId; +use chrono::Utc; +use component_manager::compute_tray_manager::{ + ComputeTrayEndpoint, ComputeTrayResult, ComputeTrayVendor, +}; +use db::machine as db_machine; +use mac_address::MacAddress; +use model::component_manager::PowerAction; +use model::machine::{ + FailureCause, FailureDetails, FailureSource, Machine, MachineMaintenanceOperation, + ManagedHostState, ManagedHostStateSnapshot, StateMachineArea, +}; +use state_controller::state_handler::{ + StateHandlerContext, StateHandlerError, StateHandlerOutcome, +}; + +use crate::context::MachineStateHandlerContextObjects; + +/// Handles the Maintenance state for a host, dispatching on the requested +/// operation (`PowerOn` / `PowerOff` / `Reset`). +pub async fn handle_maintenance( + host_machine_id: &MachineId, + mh_snapshot: &ManagedHostStateSnapshot, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> Result, StateHandlerError> { + let operation = match &mh_snapshot.managed_state { + ManagedHostState::Maintenance { operation } => *operation, + _ => unreachable!("handle_maintenance called with non-Maintenance state"), + }; + + match operation { + MachineMaintenanceOperation::PowerOn => { + handle_power_on(host_machine_id, mh_snapshot, ctx).await + } + MachineMaintenanceOperation::PowerOff => { + handle_power_off(host_machine_id, mh_snapshot, ctx).await + } + MachineMaintenanceOperation::Reset => handle_reset(host_machine_id, mh_snapshot, ctx).await, + } +} + +async fn handle_power_on( + host_machine_id: &MachineId, + mh_snapshot: &ManagedHostStateSnapshot, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> Result, StateHandlerError> { + tracing::info!(machine_id = %host_machine_id, "Machine maintenance: PowerOn"); + invoke_power_operation( + host_machine_id, + mh_snapshot, + ctx, + PowerAction::On, + "PowerOn", + ) + .await +} + +async fn handle_power_off( + host_machine_id: &MachineId, + mh_snapshot: &ManagedHostStateSnapshot, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> Result, StateHandlerError> { + tracing::info!(machine_id = %host_machine_id, "Machine maintenance: PowerOff"); + invoke_power_operation( + host_machine_id, + mh_snapshot, + ctx, + PowerAction::ForceOff, + "PowerOff", + ) + .await +} + +async fn handle_reset( + host_machine_id: &MachineId, + mh_snapshot: &ManagedHostStateSnapshot, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> Result, StateHandlerError> { + tracing::info!(machine_id = %host_machine_id, "Machine maintenance: Reset"); + invoke_power_operation( + host_machine_id, + mh_snapshot, + ctx, + PowerAction::ForceRestart, + "Reset", + ) + .await +} + +/// Common driver for component-manager-backed power maintenance operations. +async fn invoke_power_operation( + host_machine_id: &MachineId, + mh_snapshot: &ManagedHostStateSnapshot, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + action: PowerAction, + operation_label: &'static str, +) -> Result, StateHandlerError> { + let machine = &mh_snapshot.host_snapshot; + + let Some(component_manager) = ctx.services.component_manager.as_ref() else { + return finish_maintenance_with_error( + host_machine_id, + ctx, + format!( + "Machine {host_machine_id} maintenance ({operation_label}): component manager not configured" + ), + ) + .await; + }; + + let endpoint = match build_compute_tray_endpoint( + host_machine_id, + machine, + ctx.services.credential_manager.as_ref(), + ) + .await + { + Ok(endpoint) => endpoint, + Err(cause) => { + return finish_maintenance_with_error( + host_machine_id, + ctx, + format!("Machine {host_machine_id} maintenance ({operation_label}): {cause}"), + ) + .await; + } + }; + + match component_manager + .compute_tray + .power_control(std::slice::from_ref(&endpoint), action) + .await + { + Ok(results) => { + let result = results.into_iter().next().unwrap_or(ComputeTrayResult { + bmc_ip: endpoint.bmc_ip, + success: false, + error: Some("component manager returned no result".into()), + }); + + if result.success { + tracing::info!( + machine_id = %host_machine_id, + operation = operation_label, + backend = component_manager.compute_tray.name(), + "Machine power control succeeded; returning host to Ready" + ); + let mut txn = ctx.services.db_pool.begin().await?; + db_machine::clear_machine_maintenance_requested(&mut txn, *host_machine_id).await?; + return Ok(StateHandlerOutcome::transition(ManagedHostState::Ready).with_txn(txn)); + } + + let summary = result + .error + .unwrap_or_else(|| "power control failed".into()); + tracing::warn!( + machine_id = %host_machine_id, + operation = operation_label, + backend = component_manager.compute_tray.name(), + summary = %summary, + "Machine power control returned a non-success result", + ); + finish_maintenance_with_error( + host_machine_id, + ctx, + format!( + "Machine {host_machine_id} maintenance ({operation_label}): power control failed: {summary}" + ), + ) + .await + } + Err(error) => { + tracing::warn!( + machine_id = %host_machine_id, + operation = operation_label, + backend = component_manager.compute_tray.name(), + error = %error, + "Machine power control transport error", + ); + finish_maintenance_with_error( + host_machine_id, + ctx, + format!( + "Machine {host_machine_id} maintenance ({operation_label}): power control failed: {error}" + ), + ) + .await + } + } +} + +/// Build the [`ComputeTrayEndpoint`] describing this host for component manager power operations. +pub(super) async fn build_compute_tray_endpoint( + machine_id: &MachineId, + machine: &Machine, + credential_manager: &dyn CredentialManager, +) -> Result { + let bmc_mac = machine + .bmc_info + .mac + .ok_or_else(|| format!("machine {machine_id} has no BMC MAC address recorded"))?; + + let bmc_ip = machine + .bmc_info + .ip + .ok_or_else(|| format!("no BMC IP found for machine {machine_id} (bmc_mac {bmc_mac})"))?; + + let credentials = lookup_bmc_credentials(credential_manager, bmc_mac).await?; + + Ok(ComputeTrayEndpoint { + vendor: map_bmc_vendor_to_compute_tray(machine.bmc_vendor()), + bmc_ip, + bmc_credentials: credentials, + }) +} + +fn map_bmc_vendor_to_compute_tray(vendor: bmc_vendor::BMCVendor) -> ComputeTrayVendor { + match vendor { + bmc_vendor::BMCVendor::Dell => ComputeTrayVendor::Dell, + bmc_vendor::BMCVendor::Hpe => ComputeTrayVendor::Hpe, + bmc_vendor::BMCVendor::Lenovo => ComputeTrayVendor::Lenovo, + bmc_vendor::BMCVendor::Supermicro => ComputeTrayVendor::Supermicro, + bmc_vendor::BMCVendor::Nvidia => ComputeTrayVendor::Nvidia, + _ => ComputeTrayVendor::Unknown, + } +} + +async fn lookup_bmc_credentials( + credential_manager: &dyn CredentialManager, + bmc_mac: MacAddress, +) -> Result { + let bmc_key = CredentialKey::BmcCredentials { + credential_type: BmcCredentialType::BmcRoot { + bmc_mac_address: bmc_mac, + }, + }; + match credential_manager.get_credentials(&bmc_key).await { + Ok(Some(creds)) => Ok(creds), + Ok(None) => Err(format!( + "no per-device BMC credentials configured for {bmc_mac}; the device must be (re)ingested" + )), + Err(error) => Err(format!( + "failed to read BMC credentials for {bmc_mac}: {error}" + )), + } +} + +/// Clear the pending maintenance request and transition to `Failed` with the +/// given cause. Clearing the request breaks retry loops on persistent failures +/// and forces the operator to explicitly re-request maintenance to retry. +async fn finish_maintenance_with_error( + host_machine_id: &MachineId, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + cause: String, +) -> Result, StateHandlerError> { + let mut txn = ctx.services.db_pool.begin().await?; + db_machine::clear_machine_maintenance_requested(&mut txn, *host_machine_id).await?; + Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::UnhandledState { err: cause }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::MainFlow), + }, + machine_id: *host_machine_id, + retry_count: 0, + }) + .with_txn(txn)) +} + +/// If a maintenance request has been posted via `machine_maintenance_requested`, +/// transitions to [`ManagedHostState::Maintenance`] with the requested operation. +pub fn maintenance_transition_if_requested( + mh_snapshot: &ManagedHostStateSnapshot, +) -> Option> { + let req = mh_snapshot + .host_snapshot + .machine_maintenance_requested + .as_ref()?; + tracing::info!( + operation = ?req.operation, + initiator = %req.initiator, + "Machine maintenance requested; transitioning to Maintenance" + ); + Some(StateHandlerOutcome::transition( + ManagedHostState::maintenance_for_operation(req.operation), + )) +} diff --git a/crates/machine-controller/src/io.rs b/crates/machine-controller/src/io.rs index a80411c47b..8fa7dfa16d 100644 --- a/crates/machine-controller/src/io.rs +++ b/crates/machine-controller/src/io.rs @@ -27,8 +27,8 @@ use model::machine::machine_search_config::MachineSearchConfig; use model::machine::slas::MachineSlaConfig; use model::machine::{ self, AttestationMode, DpuDiscoveringState, DpuInitState, HostHealthConfig, - MachineValidatingState, ManagedHostState, ManagedHostStateSnapshot, MeasuringState, - SpdmMeasuringState, ValidationState, + MachineMaintenanceOperation, MachineValidatingState, ManagedHostState, + ManagedHostStateSnapshot, MeasuringState, SpdmMeasuringState, ValidationState, }; use sqlx::PgConnection; use state_controller::io::StateControllerIO; @@ -299,6 +299,14 @@ impl StateControllerIO for MachineStateControllerIO { ("hostnotready", machine_state_name(machine_state)) } ManagedHostState::Ready => ("ready", ""), + ManagedHostState::Maintenance { operation } => { + let op = match operation { + MachineMaintenanceOperation::PowerOn => "power_on", + MachineMaintenanceOperation::PowerOff => "power_off", + MachineMaintenanceOperation::Reset => "reset", + }; + ("maintenance", op) + } ManagedHostState::Assigned { instance_state } => { ("assigned", instance_state_name(instance_state)) } diff --git a/crates/machine-controller/tests/integration/env.rs b/crates/machine-controller/tests/integration/env.rs index 4e44c72692..1706bd0e22 100644 --- a/crates/machine-controller/tests/integration/env.rs +++ b/crates/machine-controller/tests/integration/env.rs @@ -24,8 +24,10 @@ use carbide_machine_controller::handler::{ }; use carbide_machine_controller::io::MachineStateControllerIO; use carbide_redfish::libredfish::test_support::RedfishSim; +use carbide_secrets::credentials::CredentialManager; use carbide_test_harness::prelude::*; use carbide_test_harness::{CarbideConfig, test_support}; +use component_manager::component_manager::ComponentManager; use futures::FutureExt as _; use model::machine::slas::MachineSlaConfig; use state_controller::controller::StateController; @@ -41,6 +43,8 @@ pub struct Env { pub struct EnvBuilder { pool: PgPool, runtime_config: CarbideConfig, + component_manager: Option>, + credential_manager: Option>, } impl Env { @@ -54,6 +58,8 @@ impl Env { EnvBuilder { pool, runtime_config, + component_manager: None, + credential_manager: None, } } @@ -68,10 +74,25 @@ impl EnvBuilder { self } + pub fn with_component_manager(mut self, component_manager: Arc) -> Self { + self.component_manager = Some(component_manager); + self + } + + pub fn with_credential_manager( + mut self, + credential_manager: Arc, + ) -> Self { + self.credential_manager = Some(credential_manager); + self + } + pub async fn build(self) -> Env { let Self { pool, runtime_config, + component_manager, + credential_manager, } = self; let redfish_sim = Arc::new(RedfishSim::default()); let controller_redfish_sim = redfish_sim.clone(); @@ -136,6 +157,9 @@ impl EnvBuilder { redfish_client_pool: controller_redfish_sim, ipmi_tool: carbide_ipmi::test_support(), site_config: runtime_config.machine_state_handler_site_config().into(), + component_manager, + credential_manager: credential_manager + .unwrap_or_else(|| api.credential_manager().clone()), per_object_metrics_registry, }; let machine_controller = StateController::::builder() diff --git a/crates/machine-controller/tests/integration/main.rs b/crates/machine-controller/tests/integration/main.rs index cbe97c88a2..a1aef35e66 100644 --- a/crates/machine-controller/tests/integration/main.rs +++ b/crates/machine-controller/tests/integration/main.rs @@ -18,5 +18,6 @@ // Keep the suites in one executable: sqlx-testing's migrated template is process-local. mod env; mod firmware_upgrade_completion; +mod maintenance; mod power_management; mod rack_firmware_upgrade; diff --git a/crates/machine-controller/tests/integration/maintenance.rs b/crates/machine-controller/tests/integration/maintenance.rs new file mode 100644 index 0000000000..6c580fd7c6 --- /dev/null +++ b/crates/machine-controller/tests/integration/maintenance.rs @@ -0,0 +1,456 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use std::sync::{Arc, Mutex}; + +use carbide_secrets::credentials::{CredentialManager, Credentials}; +use carbide_secrets::test_support::credentials::TestCredentialManager; +use carbide_test_harness::prelude::*; +use carbide_test_harness::test_support::fixture_config::FixtureDefault as _; +use carbide_test_support::{Case, Outcome, check_cases_async}; +use component_manager::component_manager::ComponentManager; +use component_manager::compute_tray_manager::{ + Backend, ComputeTrayEndpoint, ComputeTrayFirmwareUpdateStatus, ComputeTrayManager, + ComputeTrayResult, +}; +use component_manager::error::ComponentManagerError; +use component_manager::mock::{MockNvSwitchManager, MockPowerShelfManager}; +use component_manager::types::FirmwareUpdateOptions; +use model::component_manager::{ComputeTrayComponent, PowerAction}; +use model::machine::{ + FailureCause, FailureDetails, FailureSource, MachineMaintenanceOperation, ManagedHostState, + StateMachineArea, +}; +use model::test_support::ManagedHostConfig; + +use crate::env::Env; + +#[derive(Clone, Copy, Debug)] +enum BackendOutcome { + Success, + Empty, + NonSuccess, + TransportFailure, +} + +#[derive(Debug)] +struct ReconciliationComputeTrayManager { + outcome: Mutex, + actions: Mutex>, +} + +impl ReconciliationComputeTrayManager { + fn new() -> Self { + Self { + outcome: Mutex::new(BackendOutcome::Success), + actions: Mutex::new(Vec::new()), + } + } + + fn set_outcome(&self, outcome: BackendOutcome) { + *self.outcome.lock().unwrap() = outcome; + self.actions.lock().unwrap().clear(); + } + + fn take_actions(&self) -> Vec { + std::mem::take(&mut *self.actions.lock().unwrap()) + } +} + +#[async_trait::async_trait] +impl ComputeTrayManager for ReconciliationComputeTrayManager { + fn name(&self) -> &str { + "maintenance-test" + } + + fn backend(&self) -> Backend { + Backend::Mock + } + + async fn power_control( + &self, + endpoints: &[ComputeTrayEndpoint], + action: PowerAction, + ) -> Result, ComponentManagerError> { + self.actions.lock().unwrap().push(action); + match *self.outcome.lock().unwrap() { + BackendOutcome::Success => Ok(vec![ComputeTrayResult { + bmc_ip: endpoints[0].bmc_ip, + success: true, + error: None, + }]), + BackendOutcome::Empty => Ok(Vec::new()), + BackendOutcome::NonSuccess => Ok(vec![ComputeTrayResult { + bmc_ip: endpoints[0].bmc_ip, + success: false, + error: Some("test backend rejection".into()), + }]), + BackendOutcome::TransportFailure => Err(ComponentManagerError::Status( + tonic::Status::unavailable("test transport failure"), + )), + } + } + + async fn update_firmware( + &self, + _endpoints: &[ComputeTrayEndpoint], + _target_version: &str, + _components: &[ComputeTrayComponent], + _options: &FirmwareUpdateOptions, + ) -> Result, ComponentManagerError> { + unreachable!("firmware updates are not exercised by maintenance tests") + } + + async fn get_firmware_status( + &self, + _endpoints: &[ComputeTrayEndpoint], + ) -> Result, ComponentManagerError> { + unreachable!("firmware status is not exercised by maintenance tests") + } + + async fn list_firmware_bundles(&self) -> Result, ComponentManagerError> { + unreachable!("firmware bundles are not exercised by maintenance tests") + } +} + +#[derive(Clone, Copy, Debug)] +enum EntryState { + Ready, + Failed, +} + +#[derive(Clone, Copy, Debug)] +struct ReconciliationCase { + operation: MachineMaintenanceOperation, + entry_state: EntryState, + backend_outcome: BackendOutcome, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ResultingState { + Ready, + Failed, +} + +#[derive(Debug, PartialEq, Eq)] +struct Observation { + request_cleared: bool, + resulting_state: ResultingState, + actions: Vec, +} + +fn expected_action(operation: MachineMaintenanceOperation) -> PowerAction { + match operation { + MachineMaintenanceOperation::PowerOn => PowerAction::On, + MachineMaintenanceOperation::PowerOff => PowerAction::ForceOff, + MachineMaintenanceOperation::Reset => PowerAction::ForceRestart, + } +} + +fn component_manager(compute_tray: Arc) -> Arc { + Arc::new(ComponentManager::new( + Arc::new(MockNvSwitchManager::default()), + Arc::new(MockPowerShelfManager), + compute_tray, + false, + false, + true, + )) +} + +fn valid_credential_manager() -> Arc { + Arc::new(TestCredentialManager::new(Credentials::UsernamePassword { + username: "root".into(), + password: "password".into(), + })) +} + +async fn create_ready_host( + env: &Env, + underlay_segment: &carbide_test_harness::TestNetworkSegment, +) -> TestManagedHost { + let site_explorer = env.test_harness.default_test_site_explorer(); + let host = env + .test_harness + .managed_host_builder(&site_explorer, *underlay_segment) + .with_config(ManagedHostConfig::default()) + .build() + .await + .0; + host.advance_state(ManagedHostState::Ready).await; + host +} + +async fn enter_requested_state( + env: &Env, + host: &TestManagedHost, + operation: MachineMaintenanceOperation, + entry_state: EntryState, +) { + let mut txn = env.test_harness.db_txn().await; + let initial_state = match entry_state { + EntryState::Ready => { + db::machine::clear_failure_details(&host.host.id, &mut txn) + .await + .unwrap(); + ManagedHostState::Ready + } + EntryState::Failed => { + let details = FailureDetails { + cause: FailureCause::UnhandledState { + err: "stale failure".into(), + }, + failed_at: chrono::Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::MainFlow), + }; + db::machine::update_failure_details_by_machine_id( + &host.host.id, + &mut txn, + details.clone(), + ) + .await + .unwrap(); + ManagedHostState::Failed { + details, + machine_id: host.host.id, + retry_count: 1, + } + } + }; + + db::machine::set_machine_maintenance_requested( + &mut txn, + host.host.id, + "maintenance-test", + operation, + ) + .await + .unwrap(); + db::machine::update_state(&mut txn, &host.host.id, &initial_state) + .await + .unwrap(); + txn.commit().await.unwrap(); +} + +async fn reconcile( + env: &mut Env, + host: &TestManagedHost, + backend: &ReconciliationComputeTrayManager, + case: ReconciliationCase, +) -> Result { + backend.set_outcome(case.backend_outcome); + enter_requested_state(env, host, case.operation, case.entry_state).await; + + // First iteration accepts the request from Ready or Failed. + env.run_single_iteration().await; + let entered = host.host.machine().await; + if !matches!( + entered.state.value, + ManagedHostState::Maintenance { operation } if operation == case.operation + ) { + return Err(format!( + "request did not enter Maintenance from {:?}: {:?}", + case.entry_state, entered.state.value + )); + } + + // Second iteration reconciles the operation with the backend. + env.run_single_iteration().await; + let machine = host.host.machine().await; + let resulting_state = match machine.state.value { + ManagedHostState::Ready => ResultingState::Ready, + ManagedHostState::Failed { .. } => ResultingState::Failed, + other => return Err(format!("unexpected resulting state: {other:?}")), + }; + + Ok(Observation { + request_cleared: machine.machine_maintenance_requested.is_none(), + resulting_state, + actions: backend.take_actions(), + }) +} + +fn backend_cases() -> Vec> { + let mut cases = Vec::new(); + for operation in [ + MachineMaintenanceOperation::PowerOn, + MachineMaintenanceOperation::PowerOff, + MachineMaintenanceOperation::Reset, + ] { + for entry_state in [EntryState::Ready, EntryState::Failed] { + for backend_outcome in [ + BackendOutcome::Success, + BackendOutcome::Empty, + BackendOutcome::NonSuccess, + BackendOutcome::TransportFailure, + ] { + let succeeds = matches!(backend_outcome, BackendOutcome::Success); + cases.push(Case { + scenario: Box::leak( + format!("{operation:?} / {entry_state:?} / {backend_outcome:?}") + .into_boxed_str(), + ), + input: ReconciliationCase { + operation, + entry_state, + backend_outcome, + }, + expect: Outcome::Yields(Observation { + request_cleared: true, + resulting_state: if succeeds { + ResultingState::Ready + } else { + ResultingState::Failed + }, + actions: vec![expected_action(operation)], + }), + }); + } + } + } + cases +} + +#[sqlx_test] +async fn reconciles_all_power_operations_and_backend_outcomes(pool: PgPool) { + let backend = Arc::new(ReconciliationComputeTrayManager::new()); + let env = Env::builder(pool) + .with_component_manager(component_manager(backend.clone())) + .with_credential_manager(valid_credential_manager()) + .build() + .await; + let domain = env.test_harness.test_domain().await; + let network_controller = env.test_harness.network_controller(); + let underlay_segment = network_controller.create_underlay_segment(&domain).await; + network_controller.create_admin_segment(&domain).await; + let host = create_ready_host(&env, &underlay_segment).await; + let env = tokio::sync::Mutex::new(env); + + check_cases_async(backend_cases(), |case| { + let env = &env; + let backend = backend.clone(); + let host = &host; + async move { + let mut env = env.lock().await; + reconcile(&mut env, host, backend.as_ref(), case).await + } + }) + .await; +} + +async fn reconcile_precondition_failure( + env: &mut Env, + host: &TestManagedHost, + operation: MachineMaintenanceOperation, + entry_state: EntryState, +) -> Result { + enter_requested_state(env, host, operation, entry_state).await; + env.run_single_iteration().await; + env.run_single_iteration().await; + + let machine = host.host.machine().await; + Ok(Observation { + request_cleared: machine.machine_maintenance_requested.is_none(), + resulting_state: if matches!(machine.state.value, ManagedHostState::Failed { .. }) { + ResultingState::Failed + } else { + return Err(format!( + "unexpected resulting state: {:?}", + machine.state.value + )); + }, + actions: Vec::new(), + }) +} + +fn precondition_cases() -> Vec> +{ + let mut cases = Vec::new(); + for operation in [ + MachineMaintenanceOperation::PowerOn, + MachineMaintenanceOperation::PowerOff, + MachineMaintenanceOperation::Reset, + ] { + for entry_state in [EntryState::Ready, EntryState::Failed] { + cases.push(Case { + scenario: Box::leak(format!("{operation:?} / {entry_state:?}").into_boxed_str()), + input: (operation, entry_state), + expect: Outcome::Yields(Observation { + request_cleared: true, + resulting_state: ResultingState::Failed, + actions: Vec::new(), + }), + }); + } + } + cases +} + +#[sqlx_test] +async fn clears_requests_when_component_manager_is_missing(pool: PgPool) { + let env = Env::builder(pool) + .with_credential_manager(valid_credential_manager()) + .build() + .await; + let domain = env.test_harness.test_domain().await; + let network_controller = env.test_harness.network_controller(); + let underlay_segment = network_controller.create_underlay_segment(&domain).await; + network_controller.create_admin_segment(&domain).await; + let host = create_ready_host(&env, &underlay_segment).await; + let env = tokio::sync::Mutex::new(env); + + check_cases_async(precondition_cases(), |(operation, entry_state)| { + let env = &env; + let host = &host; + async move { + let mut env = env.lock().await; + reconcile_precondition_failure(&mut env, host, operation, entry_state).await + } + }) + .await; +} + +#[sqlx_test] +async fn clears_requests_when_credentials_are_missing(pool: PgPool) { + let backend = Arc::new(ReconciliationComputeTrayManager::new()); + let env = Env::builder(pool) + .with_component_manager(component_manager(backend.clone())) + .with_credential_manager(Arc::new(TestCredentialManager::default())) + .build() + .await; + let domain = env.test_harness.test_domain().await; + let network_controller = env.test_harness.network_controller(); + let underlay_segment = network_controller.create_underlay_segment(&domain).await; + network_controller.create_admin_segment(&domain).await; + let host = create_ready_host(&env, &underlay_segment).await; + let env = tokio::sync::Mutex::new(env); + + check_cases_async(precondition_cases(), |(operation, entry_state)| { + let env = &env; + let backend = backend.clone(); + let host = &host; + async move { + backend.set_outcome(BackendOutcome::Success); + let mut env = env.lock().await; + let mut observation = + reconcile_precondition_failure(&mut env, host, operation, entry_state).await?; + observation.actions = backend.take_actions(); + Ok(observation) + } + }) + .await; +}