fix: ignore stale resource state

server/internal/database/instance_resource.go

@@ -174,14 +174,9 @@ func (r *InstanceResource) initializeInstance(ctx context.Context, rc *resource.
 		return fmt.Errorf("failed to get TLS config: %w", err)
 	}
 
-	firstTimeSetup, err := r.isFirstTimeSetup(rc)
-	if err != nil {
-		return err
-	}
-
 	var spockSets []postgres.ReplicationSet
 	var spockTables []postgres.ReplicationSetTable
-	if r.Spec.RestoreConfig != nil && firstTimeSetup {
+	if r.Spec.RestoreConfig != nil && r.isFirstTimeSetup(rc) {
 		err = r.renameDB(ctx, tlsCfg)
 		if err != nil {
 			return fmt.Errorf("failed to rename database %q: %w", r.Spec.DatabaseName, err)
@@ -401,17 +396,11 @@ func (r *InstanceResource) dropSpock(ctx context.Context, tlsCfg *tls.Config) er
 	return nil
 }
 
-func (r *InstanceResource) isFirstTimeSetup(rc *resource.Context) (bool, error) {
+func (r *InstanceResource) isFirstTimeSetup(rc *resource.Context) bool {
 	// This instance will already exist in the state if it's been successfully
 	// created before.
-	_, err := resource.FromContext[*InstanceResource](rc, r.Identifier())
-	if errors.Is(err, resource.ErrNotFound) {
-		return true, nil
-	} else if err != nil {
-		return false, fmt.Errorf("failed to check state for previous version of this instance: %w", err)
-	}
-
-	return false, nil
+	_, ok := rc.State.Get(r.Identifier())
+	return !ok
 }
 
 func (r *InstanceResource) backupReplicationSets(

server/internal/database/node_resource.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/pgEdge/control-plane/server/internal/resource"
@@ -62,12 +63,22 @@ func (n *NodeResource) Create(ctx context.Context, rc *resource.Context) error {
 		return fmt.Errorf("node %q does not have any instances", n.Name)
 	}
 
-	// The primary instance ID should be the same on every instance
-	instance, err := resource.FromContext[*InstanceResource](rc, InstanceResourceIdentifier(n.InstanceIDs[0]))
-	if err != nil {
-		return fmt.Errorf("failed to get instance %q: %w", n.InstanceIDs[0], err)
+	// Some instances may be down or in a bad state. We'll want to check all of
+	// them to find one that knows the primary instance ID.
+	n.PrimaryInstanceID = ""
+	for _, id := range n.InstanceIDs {
+		instance, err := resource.FromContext[*InstanceResource](rc, InstanceResourceIdentifier(id))
+		if errors.Is(err, resource.ErrNotFound) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("failed to get instance %q: %w", id, err)
+		}
+
+		if instance.PrimaryInstanceID != "" {
+			n.PrimaryInstanceID = instance.PrimaryInstanceID
+			break
+		}
 	}
-	n.PrimaryInstanceID = instance.PrimaryInstanceID
 
 	return nil
 }

server/internal/resource/state.go

@@ -422,6 +422,9 @@ func FromState[T Resource](state *State, identifier Identifier) (T, error) {
 	if !ok {
 		return zero, fmt.Errorf("%w: %s", ErrNotFound, identifier.String())
 	}
+	if data.NeedsRecreate {
+		return zero, fmt.Errorf("%w: %s needs recreate", ErrNotFound, identifier.String())
+	}
 	return ToResource[T](data)
 }