From fbd0596a063a90188d1f1c7f4dc9766eef8bc6cd Mon Sep 17 00:00:00 2001 From: Vicente Ferrara Date: Mon, 20 Oct 2025 21:53:31 +0000 Subject: [PATCH] feat: Embed NetworkPolicy to extensions updated logging to avoid error rebase 'rebase --- .../api/v1alpha1/sandboxtemplate_types.go | 54 +++++ .../api/v1alpha1/zz_generated.deepcopy.go | 204 +++++++++++++++++ .../controllers/sandboxclaim_controller.go | 185 +++++++++++++++- .../sandboxclaim_controller_test.go | 205 ++++++++++++++---- extensions/examples/sandbox-claim.yaml | 12 + .../examples/secure-sandboxtemplate.yaml | 62 ++++++ ...ions.agents.x-k8s.io_sandboxtemplates.yaml | 72 ++++++ k8s/extensions-rbac.generated.yaml | 12 + 8 files changed, 765 insertions(+), 41 deletions(-) create mode 100644 extensions/examples/sandbox-claim.yaml create mode 100644 extensions/examples/secure-sandboxtemplate.yaml diff --git a/extensions/api/v1alpha1/sandboxtemplate_types.go b/extensions/api/v1alpha1/sandboxtemplate_types.go index 86f03033..94630e97 100644 --- a/extensions/api/v1alpha1/sandboxtemplate_types.go +++ b/extensions/api/v1alpha1/sandboxtemplate_types.go @@ -31,6 +31,7 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" sandboxv1alpha1 "sigs.k8s.io/agent-sandbox/api/v1alpha1" ) @@ -38,12 +39,65 @@ import ( // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. // Important: Run "make" to regenerate code after modifying this file +// NetworkPolicySpec defines the desired state of the NetworkPolicy. +type NetworkPolicySpec struct { + Enabled bool `json:"enabled,omitempty"` + IngressControllerSelectors *IngressSelector `json:"ingressControllerSelectors,omitempty"` + IngressFromIPBlocks []IPBlock `json:"ingressFromIPBlocks,omitempty"` + AdditionalIngressRules []IngressRule `json:"additionalIngressRules,omitempty"` + AdditionalEgressRules []EgressRule `json:"additionalEgressRules,omitempty"` +} + +// IngressSelector defines selectors for an in-cluster ingress controller. +type IngressSelector struct { + NamespaceSelector map[string]string `json:"namespaceSelector,omitempty"` + PodSelector map[string]string `json:"podSelector,omitempty"` +} + +// IPBlock defines a CIDR block for ingress or egress rules. +type IPBlock struct { + CIDR string `json:"cidr,omitempty"` +} + +// EgressRule defines a single egress rule. +type EgressRule struct { + Description string `json:"description,omitempty"` + ToIPBlock *IPBlockWithExcept `json:"toIPBlock,omitempty"` + ToPodSelector map[string]string `json:"toPodSelector,omitempty"` + InNamespaceSelector map[string]string `json:"inNamespaceSelector,omitempty"` + Ports []NetworkPort `json:"ports,omitempty"` +} + +// IngressRule defines a single ingress rule from another pod. +type IngressRule struct { + Description string `json:"description,omitempty"` + FromPodSelector map[string]string `json:"fromPodSelector,omitempty"` + InNamespaceSelector map[string]string `json:"inNamespaceSelector,omitempty"` +} + +// IPBlockWithExcept is for egress rules that need an "except" clause. +type IPBlockWithExcept struct { + CIDR string `json:"cidr,omitempty"` + Except []string `json:"except,omitempty"` +} + +// NetworkPort defines a port for a network policy rule. +type NetworkPort struct { + Protocol *corev1.Protocol `json:"protocol,omitempty"` + Port *int32 `json:"port,omitempty"` +} + // SandboxTemplateSpec defines the desired state of Sandbox type SandboxTemplateSpec struct { // template is the object that describes the pod spec that will be used to create // an agent sandbox. // +kubebuilder:validation:Required PodTemplate sandboxv1alpha1.PodTemplate `json:"podTemplate" protobuf:"bytes,3,opt,name=podTemplate"` + + // NetworkPolicy defines the network policy to be applied to the sandboxes + // created from this template. + // +optional + NetworkPolicy *NetworkPolicySpec `json:"networkPolicy,omitempty"` } // SandboxTemplateStatus defines the observed state of Sandbox. diff --git a/extensions/api/v1alpha1/zz_generated.deepcopy.go b/extensions/api/v1alpha1/zz_generated.deepcopy.go index 90262604..c62d54d0 100644 --- a/extensions/api/v1alpha1/zz_generated.deepcopy.go +++ b/extensions/api/v1alpha1/zz_generated.deepcopy.go @@ -5,10 +5,209 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressRule) DeepCopyInto(out *EgressRule) { + *out = *in + if in.ToIPBlock != nil { + in, out := &in.ToIPBlock, &out.ToIPBlock + *out = new(IPBlockWithExcept) + (*in).DeepCopyInto(*out) + } + if in.ToPodSelector != nil { + in, out := &in.ToPodSelector, &out.ToPodSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.InNamespaceSelector != nil { + in, out := &in.InNamespaceSelector, &out.InNamespaceSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]NetworkPort, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressRule. +func (in *EgressRule) DeepCopy() *EgressRule { + if in == nil { + return nil + } + out := new(EgressRule) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPBlock) DeepCopyInto(out *IPBlock) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPBlock. +func (in *IPBlock) DeepCopy() *IPBlock { + if in == nil { + return nil + } + out := new(IPBlock) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPBlockWithExcept) DeepCopyInto(out *IPBlockWithExcept) { + *out = *in + if in.Except != nil { + in, out := &in.Except, &out.Except + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPBlockWithExcept. +func (in *IPBlockWithExcept) DeepCopy() *IPBlockWithExcept { + if in == nil { + return nil + } + out := new(IPBlockWithExcept) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IngressRule) DeepCopyInto(out *IngressRule) { + *out = *in + if in.FromPodSelector != nil { + in, out := &in.FromPodSelector, &out.FromPodSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.InNamespaceSelector != nil { + in, out := &in.InNamespaceSelector, &out.InNamespaceSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressRule. +func (in *IngressRule) DeepCopy() *IngressRule { + if in == nil { + return nil + } + out := new(IngressRule) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IngressSelector) DeepCopyInto(out *IngressSelector) { + *out = *in + if in.NamespaceSelector != nil { + in, out := &in.NamespaceSelector, &out.NamespaceSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.PodSelector != nil { + in, out := &in.PodSelector, &out.PodSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressSelector. +func (in *IngressSelector) DeepCopy() *IngressSelector { + if in == nil { + return nil + } + out := new(IngressSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkPolicySpec) DeepCopyInto(out *NetworkPolicySpec) { + *out = *in + if in.IngressControllerSelectors != nil { + in, out := &in.IngressControllerSelectors, &out.IngressControllerSelectors + *out = new(IngressSelector) + (*in).DeepCopyInto(*out) + } + if in.IngressFromIPBlocks != nil { + in, out := &in.IngressFromIPBlocks, &out.IngressFromIPBlocks + *out = make([]IPBlock, len(*in)) + copy(*out, *in) + } + if in.AdditionalIngressRules != nil { + in, out := &in.AdditionalIngressRules, &out.AdditionalIngressRules + *out = make([]IngressRule, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AdditionalEgressRules != nil { + in, out := &in.AdditionalEgressRules, &out.AdditionalEgressRules + *out = make([]EgressRule, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkPolicySpec. +func (in *NetworkPolicySpec) DeepCopy() *NetworkPolicySpec { + if in == nil { + return nil + } + out := new(NetworkPolicySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkPort) DeepCopyInto(out *NetworkPort) { + *out = *in + if in.Protocol != nil { + in, out := &in.Protocol, &out.Protocol + *out = new(corev1.Protocol) + **out = **in + } + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkPort. +func (in *NetworkPort) DeepCopy() *NetworkPort { + if in == nil { + return nil + } + out := new(NetworkPort) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SandboxClaim) DeepCopyInto(out *SandboxClaim) { *out = *in @@ -200,6 +399,11 @@ func (in *SandboxTemplateRef) DeepCopy() *SandboxTemplateRef { func (in *SandboxTemplateSpec) DeepCopyInto(out *SandboxTemplateSpec) { *out = *in in.PodTemplate.DeepCopyInto(&out.PodTemplate) + if in.NetworkPolicy != nil { + in, out := &in.NetworkPolicy, &out.NetworkPolicy + *out = new(NetworkPolicySpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxTemplateSpec. diff --git a/extensions/controllers/sandboxclaim_controller.go b/extensions/controllers/sandboxclaim_controller.go index 378b0a0b..3f421975 100644 --- a/extensions/controllers/sandboxclaim_controller.go +++ b/extensions/controllers/sandboxclaim_controller.go @@ -18,15 +18,18 @@ import ( "context" "errors" "fmt" + "hash/fnv" "reflect" "sort" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" k8errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -38,6 +41,10 @@ import ( extensionsv1alpha1 "sigs.k8s.io/agent-sandbox/extensions/api/v1alpha1" ) +const ( + sandboxLabel = "agents.x-k8s.io/sandbox-name-hash" +) + // ErrTemplateNotFound is a sentinel error indicating a SandboxTemplate was not found. var ErrTemplateNotFound = errors.New("SandboxTemplate not found") @@ -52,6 +59,7 @@ type SandboxClaimReconciler struct { //+kubebuilder:rbac:groups=agents.x-k8s.io,resources=sandboxes,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=extensions.agents.x-k8s.io,resources=sandboxtemplates,verbs=get;list;watch //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -79,6 +87,13 @@ func (r *SandboxClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request // Try getting sandbox even if template is not found // It is possible that the template was deleted after the sandbox was created sandbox, err = r.getOrCreateSandbox(ctx, claim, template) + + if err == nil { // Only reconcile NetworkPolicy if sandbox creation was successful + if npErr := r.reconcileNetworkPolicy(ctx, claim, template); npErr != nil { + // Join the error so it gets reported in the status + err = errors.Join(err, npErr) + } + } } // Update claim status @@ -254,10 +269,10 @@ func (r *SandboxClaimReconciler) createSandbox(ctx context.Context, claim *exten Name: claim.Name, }, } - sandbox.Spec.PodTemplate = template.Spec.PodTemplate + sandbox.Spec.PodTemplate.Spec = template.Spec.PodTemplate.Spec if err := controllerutil.SetControllerReference(claim, sandbox, r.Scheme); err != nil { err = fmt.Errorf("failed to set controller reference for sandbox: %w", err) - logger.Error(err, "Error creating sandbox for claim: %q", claim.Name) + logger.Error(err, "Error setting controller reference for sandbox", "claim", claim.Name) return nil, err } @@ -278,7 +293,7 @@ func (r *SandboxClaimReconciler) createSandbox(ctx context.Context, claim *exten if err := r.Create(ctx, sandbox); err != nil { err = fmt.Errorf("sandbox create error: %w", err) - logger.Error(err, "Error creating sandbox for claim: %q", claim.Name) + logger.Error(err, "Error creating sandbox for claim", "claim", claim.Name) return nil, err } @@ -339,3 +354,167 @@ func (r *SandboxClaimReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&sandboxv1alpha1.Sandbox{}). Complete(r) } + +// reconcileNetworkPolicy ensures a NetworkPolicy exists for the claimed Sandbox, +// translating the rules from the SandboxTemplate. +func (r *SandboxClaimReconciler) reconcileNetworkPolicy(ctx context.Context, claim *extensionsv1alpha1.SandboxClaim, template *extensionsv1alpha1.SandboxTemplate) error { + logger := log.FromContext(ctx) + + // If the template doesn't exist or network policy is disabled, we do nothing. + if template == nil || template.Spec.NetworkPolicy == nil || !template.Spec.NetworkPolicy.Enabled { + logger.V(1).Info("Network policy not enabled for this template, skipping.") + // TODO: Add logic here to delete an existing NetworkPolicy if the template is changed to disabled=false. + return nil + } + + np := &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: claim.Name + "-network-policy", // A unique name for the policy + Namespace: claim.Namespace, + }, + } + + // CreateOrUpdate will create the policy if it doesn't exist, or update it + // if the template has changed. + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, np, func() error { + nameHash := NameHash(claim.Name) + podSelector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + sandboxLabel: nameHash, + }, + } + + var ingressRules []networkingv1.NetworkPolicyIngressRule + templateNP := template.Spec.NetworkPolicy + + hasIngressSources := templateNP.IngressControllerSelectors != nil || + len(templateNP.IngressFromIPBlocks) > 0 || + len(templateNP.AdditionalIngressRules) > 0 + + if hasIngressSources { + var ingressPeers []networkingv1.NetworkPolicyPeer + if sel := templateNP.IngressControllerSelectors; sel != nil { + peer := networkingv1.NetworkPolicyPeer{} + if sel.NamespaceSelector != nil { + peer.NamespaceSelector = &metav1.LabelSelector{MatchLabels: sel.NamespaceSelector} + } + if sel.PodSelector != nil { + peer.PodSelector = &metav1.LabelSelector{MatchLabels: sel.PodSelector} + } + ingressPeers = append(ingressPeers, peer) + } + for _, block := range templateNP.IngressFromIPBlocks { + ingressPeers = append(ingressPeers, networkingv1.NetworkPolicyPeer{ + IPBlock: &networkingv1.IPBlock{CIDR: block.CIDR}, + }) + } + for _, rule := range templateNP.AdditionalIngressRules { + peer := networkingv1.NetworkPolicyPeer{} + if rule.InNamespaceSelector != nil { + peer.NamespaceSelector = &metav1.LabelSelector{MatchLabels: rule.InNamespaceSelector} + } + if rule.FromPodSelector != nil { + peer.PodSelector = &metav1.LabelSelector{MatchLabels: rule.FromPodSelector} + } + ingressPeers = append(ingressPeers, peer) + } + + var ingressPorts []networkingv1.NetworkPolicyPort + if len(template.Spec.PodTemplate.Spec.Containers) > 0 { + for _, container := range template.Spec.PodTemplate.Spec.Containers { + for _, port := range container.Ports { + p := port.ContainerPort + proto := corev1.ProtocolTCP + if port.Protocol != "" { + proto = port.Protocol + } + ingressPorts = append(ingressPorts, networkingv1.NetworkPolicyPort{ + Protocol: &proto, + Port: &intstr.IntOrString{Type: intstr.Int, IntVal: p}, + }) + } + } + } + ingressRules = append(ingressRules, networkingv1.NetworkPolicyIngressRule{ + From: ingressPeers, + Ports: ingressPorts, + }) + } + + var egressRules []networkingv1.NetworkPolicyEgressRule + dnsPort53 := intstr.FromInt(53) + protoUDP := corev1.ProtocolUDP + protoTCP := corev1.ProtocolTCP + egressRules = append(egressRules, networkingv1.NetworkPolicyEgressRule{ + To: []networkingv1.NetworkPolicyPeer{{ + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"}}, + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"k8s-app": "kube-dns"}}, + }}, + Ports: []networkingv1.NetworkPolicyPort{ + {Protocol: &protoUDP, Port: &dnsPort53}, + {Protocol: &protoTCP, Port: &dnsPort53}, + }, + }) + + for _, rule := range templateNP.AdditionalEgressRules { + var egressPeers []networkingv1.NetworkPolicyPeer + var egressPorts []networkingv1.NetworkPolicyPort + if rule.ToIPBlock != nil { + egressPeers = append(egressPeers, networkingv1.NetworkPolicyPeer{ + IPBlock: &networkingv1.IPBlock{CIDR: rule.ToIPBlock.CIDR, Except: rule.ToIPBlock.Except}, + }) + } else if rule.ToPodSelector != nil { + peer := networkingv1.NetworkPolicyPeer{} + if rule.InNamespaceSelector != nil { + peer.NamespaceSelector = &metav1.LabelSelector{MatchLabels: rule.InNamespaceSelector} + } + peer.PodSelector = &metav1.LabelSelector{MatchLabels: rule.ToPodSelector} + egressPeers = append(egressPeers, peer) + } + for _, p := range rule.Ports { + portNum := intstr.FromInt(int(*p.Port)) + proto := corev1.ProtocolTCP + if p.Protocol != nil { + proto = *p.Protocol + } + egressPorts = append(egressPorts, networkingv1.NetworkPolicyPort{ + Protocol: &proto, + Port: &portNum, + }) + } + egressRules = append(egressRules, networkingv1.NetworkPolicyEgressRule{ + To: egressPeers, + Ports: egressPorts, + }) + } + + np.Spec = networkingv1.NetworkPolicySpec{ + PodSelector: podSelector, + PolicyTypes: []networkingv1.PolicyType{ + networkingv1.PolicyTypeIngress, + networkingv1.PolicyTypeEgress, + }, + Ingress: ingressRules, + Egress: egressRules, + } + + return controllerutil.SetControllerReference(claim, np, r.Scheme) + }) + + if err != nil { + logger.Error(err, "Failed to create or update NetworkPolicy for claim") + return err + } + + logger.Info("Successfully reconciled NetworkPolicy for claim", "NetworkPolicy.Name", np.Name) + return nil +} + +// NameHash generates an FNV-1a hash from a string and returns +// it as a fixed-length hexadecimal string. +func NameHash(objectName string) string { + h := fnv.New32a() + h.Write([]byte(objectName)) + hashValue := h.Sum32() + return fmt.Sprintf("%08x", hashValue) +} diff --git a/extensions/controllers/sandboxclaim_controller_test.go b/extensions/controllers/sandboxclaim_controller_test.go index c9737a3c..ddae93bf 100644 --- a/extensions/controllers/sandboxclaim_controller_test.go +++ b/extensions/controllers/sandboxclaim_controller_test.go @@ -22,6 +22,7 @@ import ( "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" k8errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -55,6 +56,51 @@ func TestSandboxClaimReconcile(t *testing.T) { }, }, } + + // New template with NetworkPolicy enabled for testing that feature. + templateWithNP := &extensionsv1alpha1.SandboxTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-template-with-np", + Namespace: "default", + }, + Spec: extensionsv1alpha1.SandboxTemplateSpec{ + PodTemplate: sandboxv1alpha1.PodTemplate{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test-container", + Image: "test-image", + Ports: []corev1.ContainerPort{{ContainerPort: 8080}}, + }, + }, + }, + }, + NetworkPolicy: &extensionsv1alpha1.NetworkPolicySpec{ + Enabled: true, + IngressControllerSelectors: &extensionsv1alpha1.IngressSelector{ + NamespaceSelector: map[string]string{"ns-role": "ingress"}, + PodSelector: map[string]string{"app": "ingress"}, + }, + AdditionalEgressRules: []extensionsv1alpha1.EgressRule{ + { + Description: "Allow to metrics", + ToPodSelector: map[string]string{ + "app": "metrics", + }, + InNamespaceSelector: map[string]string{ + "ns-role": "monitoring", + }, + }, + }, + }, + }, + } + + // New template with NetworkPolicy disabled. + templateWithNPDisabled := templateWithNP.DeepCopy() + templateWithNPDisabled.Name = "test-template-np-disabled" + templateWithNPDisabled.Spec.NetworkPolicy.Enabled = false + claim := &extensionsv1alpha1.SandboxClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "test-claim", @@ -110,16 +156,20 @@ func TestSandboxClaimReconcile(t *testing.T) { } testCases := []struct { - name string - existingObjects []client.Object - expectSandbox bool - expectError bool - expectedCondition metav1.Condition + name string + claimToReconcile *extensionsv1alpha1.SandboxClaim + existingObjects []client.Object + expectSandbox bool + expectNetworkPolicy bool + expectError bool + expectedCondition metav1.Condition + validateNetworkPolicy func(t *testing.T, np *networkingv1.NetworkPolicy) }{ { - name: "sandbox is created when a claim is made", - existingObjects: []client.Object{template, claim}, - expectSandbox: true, + name: "sandbox is created when a claim is made", + claimToReconcile: claim, + existingObjects: []client.Object{template}, // FIX: Removed duplicate 'claim' + expectSandbox: true, expectedCondition: metav1.Condition{ Type: string(sandboxv1alpha1.SandboxConditionReady), Status: metav1.ConditionFalse, @@ -128,10 +178,11 @@ func TestSandboxClaimReconcile(t *testing.T) { }, }, { - name: "sandbox is not created when template is not found", - existingObjects: []client.Object{claim}, - expectSandbox: false, - expectError: true, + name: "sandbox is not created when template is not found", + claimToReconcile: claim, + existingObjects: []client.Object{}, // FIX: Removed duplicate 'claim' + expectSandbox: false, + expectError: true, expectedCondition: metav1.Condition{ Type: string(sandboxv1alpha1.SandboxConditionReady), Status: metav1.ConditionFalse, @@ -140,10 +191,11 @@ func TestSandboxClaimReconcile(t *testing.T) { }, }, { - name: "sandbox exists but is not controlled by claim", - existingObjects: []client.Object{template, claim, uncontrolledSandbox}, - expectSandbox: true, - expectError: true, + name: "sandbox exists but is not controlled by claim", + claimToReconcile: claim, + existingObjects: []client.Object{template, uncontrolledSandbox}, // FIX: Removed duplicate 'claim' + expectSandbox: true, + expectError: true, expectedCondition: metav1.Condition{ Type: string(sandboxv1alpha1.SandboxConditionReady), Status: metav1.ConditionFalse, @@ -152,9 +204,10 @@ func TestSandboxClaimReconcile(t *testing.T) { }, }, { - name: "sandbox exists and is controlled by claim", - existingObjects: []client.Object{template, claim, controlledSandbox}, - expectSandbox: true, + name: "sandbox exists and is controlled by claim", + claimToReconcile: claim, + existingObjects: []client.Object{template, controlledSandbox}, // FIX: Removed duplicate 'claim' + expectSandbox: true, expectedCondition: metav1.Condition{ Type: string(sandboxv1alpha1.SandboxConditionReady), Status: metav1.ConditionFalse, @@ -163,9 +216,10 @@ func TestSandboxClaimReconcile(t *testing.T) { }, }, { - name: "sandbox exists but template is not found", - existingObjects: []client.Object{claim, readySandbox}, - expectSandbox: true, + name: "sandbox exists but template is not found", + claimToReconcile: claim, + existingObjects: []client.Object{readySandbox}, // FIX: Removed duplicate 'claim' + expectSandbox: true, expectedCondition: metav1.Condition{ Type: string(sandboxv1alpha1.SandboxConditionReady), Status: metav1.ConditionTrue, @@ -174,9 +228,10 @@ func TestSandboxClaimReconcile(t *testing.T) { }, }, { - name: "sandbox is ready", - existingObjects: []client.Object{template, claim, readySandbox}, - expectSandbox: true, + name: "sandbox is ready", + claimToReconcile: claim, + existingObjects: []client.Object{template, readySandbox}, + expectSandbox: true, expectedCondition: metav1.Condition{ Type: string(sandboxv1alpha1.SandboxConditionReady), Status: metav1.ConditionTrue, @@ -184,20 +239,82 @@ func TestSandboxClaimReconcile(t *testing.T) { Message: "Sandbox is ready", }, }, + { + name: "sandbox is created with network policy enabled", + claimToReconcile: &extensionsv1alpha1.SandboxClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "test-claim-np", Namespace: "default", UID: "claim-np-uid"}, + Spec: extensionsv1alpha1.SandboxClaimSpec{TemplateRef: extensionsv1alpha1.SandboxTemplateRef{Name: "test-template-with-np"}}, + }, + existingObjects: []client.Object{templateWithNP}, + expectSandbox: true, + expectNetworkPolicy: true, + expectedCondition: metav1.Condition{ + Type: string(sandboxv1alpha1.SandboxConditionReady), + Status: metav1.ConditionFalse, + Reason: "SandboxNotReady", + Message: "Sandbox is not ready", + }, + validateNetworkPolicy: func(t *testing.T, np *networkingv1.NetworkPolicy) { + // 1. Check Owner Reference + if diff := cmp.Diff(np.OwnerReferences[0].UID, types.UID("claim-np-uid")); diff != "" { + t.Errorf("unexpected owner reference UID:\n%s", diff) + } + // 2. Check Pod Selector + expectedHash := NameHash("test-claim-np") + if diff := cmp.Diff(np.Spec.PodSelector.MatchLabels[sandboxLabel], expectedHash); diff != "" { + t.Errorf("unexpected pod selector hash:\n%s", diff) + } + // 3. Check Ingress Rule Translation + if len(np.Spec.Ingress) != 1 { + t.Fatalf("expected 1 ingress rule, got %d", len(np.Spec.Ingress)) + } + ingressRule := np.Spec.Ingress[0] + if diff := cmp.Diff(ingressRule.From[0].NamespaceSelector.MatchLabels, map[string]string{"ns-role": "ingress"}); diff != "" { + t.Errorf("unexpected ingress namespace selector:\n%s", diff) + } + // 4. Check Egress Rule Translation + if len(np.Spec.Egress) != 2 { // 1 for DNS + 1 custom + t.Fatalf("expected 2 egress rules, got %d", len(np.Spec.Egress)) + } + egressRule := np.Spec.Egress[1] // Check the custom rule + if diff := cmp.Diff(egressRule.To[0].PodSelector.MatchLabels, map[string]string{"app": "metrics"}); diff != "" { + t.Errorf("unexpected egress pod selector:\n%s", diff) + } + }, + }, + { + name: "sandbox is created with network policy disabled", + claimToReconcile: &extensionsv1alpha1.SandboxClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "test-claim-np-disabled", Namespace: "default", UID: "claim-np-disabled-uid"}, + Spec: extensionsv1alpha1.SandboxClaimSpec{TemplateRef: extensionsv1alpha1.SandboxTemplateRef{Name: "test-template-np-disabled"}}, + }, + existingObjects: []client.Object{templateWithNPDisabled}, + expectSandbox: true, + expectNetworkPolicy: false, // Should not create a NetworkPolicy + expectedCondition: metav1.Condition{ + Type: string(sandboxv1alpha1.SandboxConditionReady), + Status: metav1.ConditionFalse, + Reason: "SandboxNotReady", + Message: "Sandbox is not ready", + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { scheme := newScheme(t) - client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tc.existingObjects...).WithStatusSubresource(claim).Build() + // Add the claim we are reconciling to the list of existing objects + allObjects := append(tc.existingObjects, tc.claimToReconcile) + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(allObjects...).WithStatusSubresource(tc.claimToReconcile).Build() + reconciler := &SandboxClaimReconciler{ Client: client, Scheme: scheme, } req := reconcile.Request{ NamespacedName: types.NamespacedName{ - Name: "test-claim", - Namespace: "default", + Name: tc.claimToReconcile.Name, + Namespace: tc.claimToReconcile.Namespace, }, } _, err := reconciler.Reconcile(context.Background(), req) @@ -217,10 +334,18 @@ func TestSandboxClaimReconcile(t *testing.T) { t.Fatalf("expected sandbox to not exist, but got err: %v", err) } - if tc.expectSandbox { - if diff := cmp.Diff(sandbox.Spec.PodTemplate.Spec, template.Spec.PodTemplate.Spec); diff != "" { - t.Errorf("unexpected sandbox spec:\n%s", diff) - } + // Validate NetworkPolicy + var np networkingv1.NetworkPolicy + npName := types.NamespacedName{Name: req.Name + "-network-policy", Namespace: req.Namespace} + err = client.Get(context.Background(), npName, &np) + if tc.expectNetworkPolicy && err != nil { + t.Fatalf("get network policy: (%v)", err) + } + if !tc.expectNetworkPolicy && !k8errors.IsNotFound(err) { + t.Fatalf("expected network policy to not exist, but got err: %v", err) + } + if tc.validateNetworkPolicy != nil { + tc.validateNetworkPolicy(t, &np) } var updatedClaim extensionsv1alpha1.SandboxClaim @@ -236,9 +361,10 @@ func TestSandboxClaimReconcile(t *testing.T) { if condition.Reason != "ReconcilerError" { t.Errorf("expected condition reason %q, got %q", "ReconcilerError", condition.Reason) } - } - if diff := cmp.Diff(tc.expectedCondition, condition, cmp.Comparer(ignoreTimestamp)); diff != "" { - t.Errorf("unexpected condition:\n%s", diff) + } else { // Only do a full diff if not expecting a generic reconciler error + if diff := cmp.Diff(tc.expectedCondition, condition, cmp.Comparer(ignoreTimestamp)); diff != "" { + t.Errorf("unexpected condition:\n%s", diff) + } } }) } @@ -486,13 +612,16 @@ func TestSandboxClaimPodAdoption(t *testing.T) { func newScheme(t *testing.T) *runtime.Scheme { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("reconcile: (%v)", err) + t.Fatalf("add to scheme: (%v)", err) } if err := extensionsv1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("reconcile: (%v)", err) + t.Fatalf("add to scheme: (%v)", err) } if err := corev1.AddToScheme(scheme); err != nil { - t.Fatalf("reconcile: (%v)", err) + t.Fatalf("add to scheme: (%v)", err) + } + if err := networkingv1.AddToScheme(scheme); err != nil { + t.Fatalf("add to scheme: (%v)", err) } return scheme } diff --git a/extensions/examples/sandbox-claim.yaml b/extensions/examples/sandbox-claim.yaml new file mode 100644 index 00000000..d0339b16 --- /dev/null +++ b/extensions/examples/sandbox-claim.yaml @@ -0,0 +1,12 @@ +apiVersion: extensions.agents.x-k8s.io/v1alpha1 +kind: SandboxClaim +metadata: + # This will be the name of your sandbox pod and its associated resources. + name: my-secure-sandbox + # This MUST be in the same namespace as the SandboxTemplate. + namespace: default +spec: + # It points to the template. + sandboxTemplateRef: + # This name must exactly match the 'metadata.name' of your SandboxTemplate. + name: secure-datascience-template \ No newline at end of file diff --git a/extensions/examples/secure-sandboxtemplate.yaml b/extensions/examples/secure-sandboxtemplate.yaml new file mode 100644 index 00000000..674234e8 --- /dev/null +++ b/extensions/examples/secure-sandboxtemplate.yaml @@ -0,0 +1,62 @@ +# How This Template Fulfills Your Requirements: +# Restrict Pod-to-Pod Communication: Because enabled is true, your controller will create a unique NetworkPolicy for each sandbox. +# This individual policy is what guarantees that sandbox-1 cannot talk to sandbox-2. + +# Only Allow Pod Ingress from MST L7XLB: The ingressControllerSelectors block explicitly defines the L7XLB's labels as the only allowed ingress source. + +# Only Allow Egress if Required: The controller adds a DNS rule by default. +# creates a rule to allow TCP and UDP traffic on port 53 to the pods labeled k8s-app: kube-dns in the kube-system namespace. +# By providing an empty additionalEgressRules: [], you are declaring that no other egress is required, so everything else will be blocked. + +# Don't Allow Access to API Server: This is a direct result of the "default-deny" egress policy. Since there is no rule in additionalEgressRules that allows traffic to the API server, it is blocked. +apiVersion: extensions.agents.x-k8s.io/v1alpha1 +kind: SandboxTemplate +metadata: + name: secure-datascience-template + namespace: default +spec: + podTemplate: + spec: + # This pod uses a non-root user and gVisor for runtime sandboxing. + runtimeClassName: gvisor + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + runAsNonRoot: true + containers: + - name: my-container + image: busybox + command: ["/bin/sh", "-c", "sleep 36000"] + ports: + - containerPort: 8888 + protocol: TCP + + + # 2. Define the Network Policy to enforce the security requirements. + networkPolicy: + # This enables the "default-deny" NetworkPolicy creation. + enabled: true + + # REQUIREMENT #3: Only allow pod ingress from MST L7XLB. + # This section defines the trusted ingress source. Traffic from any + # other source, including other sandbox pods, will be blocked. + ingressControllerSelectors: + namespaceSelector: + # Example: The L7XLB pods are in a namespace with this label + istio-injection: enabled + podSelector: + # Example: The L7XLB pods have this label + app: istio-ingressgateway + + # For an external L7XLB, we use 'ingressFromIPBlocks' to specify + # its known internal IP range as the trusted source. + #ingressFromIPBlocks: + # - cidr: "10.200.10.0/24" # Example: The known internal IP range of your external L7XLB + + # REQUIREMENT #2 & #4: Only allow egress if required & block API server access. + # The controller automatically adds a rule to allow DNS. By providing + # an empty 'additionalEgressRules' list, we are explicitly stating that + # no other outgoing traffic is required or permitted. This blocks all + # other egress, including to the Kubernetes API server and other pods. + additionalEgressRules: [] diff --git a/k8s/crds/extensions.agents.x-k8s.io_sandboxtemplates.yaml b/k8s/crds/extensions.agents.x-k8s.io_sandboxtemplates.yaml index 3123753b..5058a263 100644 --- a/k8s/crds/extensions.agents.x-k8s.io_sandboxtemplates.yaml +++ b/k8s/crds/extensions.agents.x-k8s.io_sandboxtemplates.yaml @@ -28,6 +28,78 @@ spec: type: object spec: properties: + networkPolicy: + properties: + additionalEgressRules: + items: + properties: + description: + type: string + inNamespaceSelector: + additionalProperties: + type: string + type: object + ports: + items: + properties: + port: + format: int32 + type: integer + protocol: + type: string + type: object + type: array + toIPBlock: + properties: + cidr: + type: string + except: + items: + type: string + type: array + type: object + toPodSelector: + additionalProperties: + type: string + type: object + type: object + type: array + additionalIngressRules: + items: + properties: + description: + type: string + fromPodSelector: + additionalProperties: + type: string + type: object + inNamespaceSelector: + additionalProperties: + type: string + type: object + type: object + type: array + enabled: + type: boolean + ingressControllerSelectors: + properties: + namespaceSelector: + additionalProperties: + type: string + type: object + podSelector: + additionalProperties: + type: string + type: object + type: object + ingressFromIPBlocks: + items: + properties: + cidr: + type: string + type: object + type: array + type: object podTemplate: properties: metadata: diff --git a/k8s/extensions-rbac.generated.yaml b/k8s/extensions-rbac.generated.yaml index 80e0d90a..87a96058 100644 --- a/k8s/extensions-rbac.generated.yaml +++ b/k8s/extensions-rbac.generated.yaml @@ -58,3 +58,15 @@ rules: - get - list - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch