Skip to content

Commit c4dbb60

Browse files
kartikjoshi21kartikjoshi21
committed
minikube: Add ipv6 support for docker driver and calico cni
Signed-off-by: Kartik Joshi <[email protected]>
1 parent 75c1811 commit c4dbb60

File tree

14 files changed

+653
-104
lines changed

14 files changed

+653
-104
lines changed

‎cmd/minikube/cmd/start_flags.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -565,11 +565,20 @@ func normalizeAndValidateIPFamily(cc *config.ClusterConfig) {
565565
}
566566

567567
if s := cc.KubernetesConfig.PodCIDRv6; s != "" {
568-
if _, _, err := net.ParseCIDR(s); err != nil {
569-
exit.Message(reason.Usage, "--pod-cidr-v6 must be a valid IPv6 CIDR: {{.e}}", out.V{"e": err})
570-
}
568+
if _, _, err := net.ParseCIDR(s); err != nil {
569+
exit.Message(reason.Usage, "--pod-cidr-v6 must be a valid IPv6 CIDR: {{.e}}", out.V{"e": err})
570+
}
571571
}
572572

573+
// validate static IPv6 if provided
574+
if s := cc.StaticIPv6; s != "" {
575+
ip := net.ParseIP(s)
576+
if ip == nil || ip.To4() != nil {
577+
exit.Message(reason.Usage, "--static-ipv6 must be a valid IPv6 address")
578+
}
579+
}
580+
581+
573582
// Docker driver guardrails: Linux daemon + IPv6 must be enabled
574583
if driver.IsDocker(cc.Driver) && fam != "ipv4" {
575584
// Desktop vs Linux daemon hint (we can't reliably detect IPv6 enabled here)

‎pkg/drivers/kic/oci/network.go‎

Lines changed: 61 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -88,13 +88,39 @@ func RoutableHostIPFromInside(ociBin string, clusterName string, containerName s
8888
// digDNS will get the IP record for a dns
8989
func digDNS(ociBin, containerName, dns string) (net.IP, error) {
9090
rr, err := runCmd(exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns))
91-
ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String()))
92-
if err != nil {
93-
return ip, errors.Wrapf(err, "resolve dns to ip")
94-
}
95-
96-
klog.Infof("got host ip for mount in container by digging dns: %s", ip.String())
97-
return ip, nil
91+
if err != nil {
92+
// still try to parse whatever output we got
93+
klog.Infof("dig returned error, attempting to parse output anyway: %v", err)
94+
}
95+
out := strings.TrimSpace(rr.Stdout.String())
96+
if out == "" {
97+
return nil, errors.Wrapf(err, "resolve dns to ip")
98+
}
99+
// Parse line-by-line. On non-Linux (Docker Desktop), prefer IPv4 for better routability.
100+
var firstIP net.IP
101+
for _, line := range strings.Split(out, "\n") {
102+
s := strings.TrimSpace(line)
103+
if s == "" {
104+
continue
105+
}
106+
ip := net.ParseIP(s)
107+
if ip == nil {
108+
continue
109+
}
110+
if runtime.GOOS != "linux" && ip.To4() == nil {
111+
// Prefer IPv4 on Desktop; keep looking for an A record
112+
if firstIP == nil { firstIP = ip }
113+
continue
114+
}
115+
klog.Infof("got host ip for mount in container by digging dns: %s", ip.String())
116+
return ip, nil
117+
}
118+
// Fallback: return first valid IP if only AAAA answers were present
119+
if firstIP != nil {
120+
klog.Infof("got host ip for mount in container by digging dns (first match): %s", firstIP.String())
121+
return firstIP, nil
122+
}
123+
return nil, errors.New("no A/AAAA answers returned by dig")
98124
}
99125

100126
// gatewayIP inspects oci container to find a gateway IP string
@@ -104,8 +130,16 @@ func gatewayIP(ociBin, containerName string) (string, error) {
104130
return "", errors.Wrapf(err, "inspect gateway")
105131
}
106132
if gatewayIP := strings.TrimSpace(rr.Stdout.String()); gatewayIP != "" {
107-
return gatewayIP, nil
108-
}
133+
return gatewayIP, nil
134+
}
135+
136+
// Fallback to IPv6 gateway (needed for IPv6-only / dual-stack)
137+
rr6, err6 := runCmd(exec.Command(ociBin, "container", "inspect", "--format", "{{.NetworkSettings.IPv6Gateway}}", containerName))
138+
if err6 == nil {
139+
if gatewayIP6 := strings.TrimSpace(rr6.Stdout.String()); gatewayIP6 != "" {
140+
return gatewayIP6, nil
141+
}
142+
}
109143

110144
// https://github.com/kubernetes/minikube/issues/11293
111145
// need to check nested network
@@ -126,16 +160,24 @@ func gatewayIP(ociBin, containerName string) (string, error) {
126160
}
127161

128162
func networkGateway(ociBin, container, network string) (string, error) {
129-
format := fmt.Sprintf(`
130-
{{ if index .NetworkSettings.Networks %q}}
131-
{{(index .NetworkSettings.Networks %q).Gateway}}
132-
{{ end }}
133-
`, network, network)
134-
rr, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", format, container))
163+
// First try IPv4 gateway on the specific network
164+
format4 := fmt.Sprintf(`{{ if index .NetworkSettings.Networks %q}}{{(index .NetworkSettings.Networks %q).Gateway}}{{ end }}`, network, network)
165+
rr, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", format4, container))
135166
if err != nil {
136167
return "", errors.Wrapf(err, "inspect gateway")
137168
}
138-
return strings.TrimSpace(rr.Stdout.String()), nil
169+
170+
gw := strings.TrimSpace(rr.Stdout.String())
171+
if gw != "" {
172+
return gw, nil
173+
}
174+
// Fallback to IPv6 gateway
175+
format6 := fmt.Sprintf(`{{ if index .NetworkSettings.Networks %q}}{{(index .NetworkSettings.Networks %q).IPv6Gateway}}{{ end }}`, network, network)
176+
rr6, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", format6, container))
177+
if err != nil {
178+
return "", errors.Wrapf(err, "inspect ipv6 gateway")
179+
}
180+
return strings.TrimSpace(rr6.Stdout.String()), nil
139181
}
140182

141183
// containerGatewayIP gets the default gateway ip for the container
@@ -188,10 +230,9 @@ func ForwardedPort(ociBin string, ociID string, contPort int) (int, error) {
188230
o := strings.TrimSpace(rr.Stdout.String())
189231
o = strings.Trim(o, "'")
190232
p, err := strconv.Atoi(o)
191-
192-
if err != nil {
193-
return p, errors.Wrapf(err, "convert host-port %q to number", p)
194-
}
233+
if err != nil {
234+
return 0, errors.Wrapf(err, "convert host-port %q to number", o)
235+
}
195236

196237
return p, nil
197238
}

‎pkg/drivers/kic/oci/network_create.go‎

Lines changed: 146 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -76,66 +76,159 @@ func CreateNetwork(ociBin, networkName, subnet, staticIP string) (net.IP, error)
7676
return CreateNetworkWithIPFamily(ociBin, networkName, subnet, "", staticIP, "", "ipv4")
7777
}
7878

79-
func CreateNetworkWithIPFamily(ociBin, networkName, subnet, subnetv6, staticIP, staticIPv6, ipFamily string) (net.IP, error) {
80-
bridgeName := defaultBridgeName(ociBin)
81-
if networkName == bridgeName {
82-
klog.Infof("skipping creating network since default network %s was specified", networkName)
83-
return nil, nil
84-
}
8579

86-
// For IPv6-only or dual-stack networks, use the v6/dual creator
87-
if ipFamily == "ipv6" || ipFamily == "dual" {
88-
return createV6OrDualNetwork(ociBin, networkName, subnet, subnetv6, ipFamily)
89-
}
90-
91-
// check if the network already exists
92-
info, err := containerNetworkInspect(ociBin, networkName)
93-
if err == nil {
94-
klog.Infof("Found existing network %+v", info)
95-
return info.gateway, nil
96-
}
80+
func CreateNetworkWithIPFamily(ociBin, networkName, subnet, subnetv6, staticIP, staticIPv6, ipFamily string) (net.IP, error) {
81+
bridgeName := defaultBridgeName(ociBin)
82+
if networkName == bridgeName {
83+
klog.Infof("skipping creating network since default network %s was specified", networkName)
84+
return nil, nil
85+
}
9786

98-
// will try to get MTU from the docker network to avoid issue with systems with exotic MTU settings.
99-
// related issue #9528
100-
info, err = containerNetworkInspect(ociBin, bridgeName)
101-
if err != nil {
102-
klog.Warningf("failed to get mtu information from the %s's default network %q: %v", ociBin, bridgeName, err)
103-
}
87+
// If already exists, reuse.
88+
if info, err := containerNetworkInspect(ociBin, networkName); err == nil {
89+
klog.Infof("Found existing network %+v", info)
90+
return info.gateway, nil
91+
}
10492

105-
tries := 20
93+
// Learn MTU from the default bridge (best effort)
94+
bridgeInfo, berr := containerNetworkInspect(ociBin, bridgeName)
95+
if berr != nil {
96+
klog.Warningf("failed to get mtu information from the %s's default network %q: %v", ociBin, bridgeName, berr)
97+
}
98+
// ----- IPv6/dual flow -----
99+
if ipFamily == "ipv6" || ipFamily == "dual" {
100+
// Decide v6 subnet
101+
if subnetv6 == "" {
102+
if staticIPv6 != "" {
103+
if s, err := cidr64ForIP(staticIPv6); err == nil {
104+
subnetv6 = s
105+
} else {
106+
return nil, errors.Wrap(err, "derive /64 from --static-ipv6")
107+
}
108+
} else {
109+
subnetv6 = firstSubnetAddrv6(subnetv6) // default fd00::/64
110+
}
111+
}
112+
113+
// Build args; enable IPv6 always in this branch
114+
args := []string{"network", "create", "--driver=bridge", "--ipv6"}
115+
116+
// For dual, also choose a free IPv4 subnet similar to the v4-only flow
117+
if ipFamily == "dual" {
118+
tries := 20
119+
start := firstSubnetAddr(subnet)
120+
if staticIP != "" {
121+
tries = 1
122+
start = staticIP
123+
}
124+
125+
// Try up to 5 candidate /24s starting at start, stepping by 9 (as before)
126+
var lastErr error
127+
for attempts, subnetAddr := 0, start; attempts < 5; attempts++ {
128+
var p *network.Parameters
129+
p, lastErr = network.FreeSubnet(subnetAddr, 9, tries)
130+
if lastErr != nil {
131+
klog.Errorf("failed to find free IPv4 subnet for %s network %s after %d attempts: %v", ociBin, networkName, 20, lastErr)
132+
return nil, fmt.Errorf("un-retryable: %w", lastErr)
133+
}
134+
135+
argsWithV4 := append([]string{}, args...)
136+
argsWithV4 = append(argsWithV4, "--subnet", p.CIDR, "--gateway", p.Gateway)
137+
argsWithV4 = append(argsWithV4, "--subnet", subnetv6)
138+
if ociBin == Docker && bridgeInfo.mtu > 0 {
139+
argsWithV4 = append(argsWithV4, "-o", fmt.Sprintf("com.docker.network.driver.mtu=%d", bridgeInfo.mtu))
140+
}
141+
argsWithV4 = append(argsWithV4,
142+
fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"),
143+
fmt.Sprintf("--label=%s=%s", ProfileLabelKey, networkName),
144+
networkName,
145+
)
146+
147+
rr, err := runCmd(exec.Command(ociBin, argsWithV4...))
148+
if err == nil {
149+
ni, _ := containerNetworkInspect(ociBin, networkName)
150+
return ni.gateway, nil
151+
}
152+
153+
out := rr.Output()
154+
// Respect same retry conditions as v4-only
155+
if strings.Contains(out, "Pool overlaps") ||
156+
(strings.Contains(out, "failed to allocate gateway") && strings.Contains(out, "Address already in use")) ||
157+
strings.Contains(out, "is being used by a network interface") ||
158+
strings.Contains(out, "is already used on the host or by another config") {
159+
klog.Warningf("failed to create %s network %s %s (dual): %v; retrying with next IPv4 subnet", ociBin, networkName, p.CIDR, err)
160+
subnetAddr = p.IP
161+
continue
162+
}
163+
// Non-retryable
164+
klog.Errorf("error creating dual-stack network %s: %v", networkName, err)
165+
return nil, fmt.Errorf("un-retryable: %w", err)
166+
}
167+
return nil, fmt.Errorf("failed to create %s network %s (dual): %w", ociBin, networkName, lastErr)
168+
}
169+
170+
// ipv6-only (no IPv4)
171+
args = append(args, "--subnet", subnetv6)
172+
if ociBin == Docker && bridgeInfo.mtu > 0 {
173+
args = append(args, "-o", fmt.Sprintf("com.docker.network.driver.mtu=%d", bridgeInfo.mtu))
174+
}
175+
args = append(args,
176+
fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"),
177+
fmt.Sprintf("--label=%s=%s", ProfileLabelKey, networkName),
178+
networkName,
179+
)
180+
181+
if _, err := runCmd(exec.Command(ociBin, args...)); err != nil {
182+
klog.Warningf("failed to create %s network %q (ipv6-only): %v", ociBin, networkName, err)
183+
return nil, fmt.Errorf("create %s network %q: %w", ociBin, networkName, err)
184+
}
185+
ni, _ := containerNetworkInspect(ociBin, networkName)
186+
return ni.gateway, nil
187+
}
106188

107-
// we don't want to increment the subnet IP on network creation failure if the user specifies a static IP, so set tries to 1
108-
if staticIP != "" {
109-
tries = 1
110-
subnet = staticIP
111-
}
189+
// ----- IPv4-only flow (existing logic) -----
190+
// keep current implementation
191+
tries := 20
192+
if staticIP != "" {
193+
tries = 1
194+
subnet = staticIP
195+
}
196+
var lastErr error
197+
for attempts, subnetAddr := 0, firstSubnetAddr(subnet); attempts < 5; attempts++ {
198+
var p *network.Parameters
199+
p, lastErr = network.FreeSubnet(subnetAddr, 9, tries)
200+
if lastErr != nil {
201+
klog.Errorf("failed to find free subnet for %s network %s after %d attempts: %v", ociBin, networkName, 20, lastErr)
202+
return nil, fmt.Errorf("un-retryable: %w", lastErr)
203+
}
204+
gw, err := tryCreateDockerNetwork(ociBin, p, bridgeInfo.mtu, networkName)
205+
if err == nil {
206+
klog.Infof("%s network %s %s created", ociBin, networkName, p.CIDR)
207+
return gw, nil
208+
}
209+
if !errors.Is(err, ErrNetworkSubnetTaken) && !errors.Is(err, ErrNetworkGatewayTaken) {
210+
klog.Errorf("error while trying to create %s network %s %s: %v", ociBin, networkName, p.CIDR, err)
211+
return nil, fmt.Errorf("un-retryable: %w", err)
212+
}
213+
klog.Warningf("failed to create %s network %s %s, will retry: %v", ociBin, networkName, p.CIDR, err)
214+
subnetAddr = p.IP
215+
}
216+
return nil, fmt.Errorf("failed to create %s network %s: %w", ociBin, networkName, lastErr)
217+
218+
}
112219

113-
// retry up to 5 times to create container network
114-
for attempts, subnetAddr := 0, firstSubnetAddr(subnet); attempts < 5; attempts++ {
115-
// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
116-
// will be like 192.168.49.0/24,..., 192.168.220.0/24 (in increment steps of 9)
117-
var subnet *network.Parameters
118-
subnet, err = network.FreeSubnet(subnetAddr, 9, tries)
119-
if err != nil {
120-
klog.Errorf("failed to find free subnet for %s network %s after %d attempts: %v", ociBin, networkName, 20, err)
121-
return nil, fmt.Errorf("un-retryable: %w", err)
122-
}
123-
info.gateway, err = tryCreateDockerNetwork(ociBin, subnet, info.mtu, networkName)
124-
if err == nil {
125-
klog.Infof("%s network %s %s created", ociBin, networkName, subnet.CIDR)
126-
return info.gateway, nil
127-
}
128-
// don't retry if error is not address is taken
129-
if !errors.Is(err, ErrNetworkSubnetTaken) && !errors.Is(err, ErrNetworkGatewayTaken) {
130-
klog.Errorf("error while trying to create %s network %s %s: %v", ociBin, networkName, subnet.CIDR, err)
131-
return nil, fmt.Errorf("un-retryable: %w", err)
132-
}
133-
klog.Warningf("failed to create %s network %s %s, will retry: %v", ociBin, networkName, subnet.CIDR, err)
134-
subnetAddr = subnet.IP
135-
}
136-
return info.gateway, fmt.Errorf("failed to create %s network %s: %w", ociBin, networkName, err)
220+
// cidr64ForIP returns a /64 CIDR string covering the provided IPv6 address.
221+
func cidr64ForIP(ipStr string) (string, error) {
222+
ip := net.ParseIP(ipStr)
223+
if ip == nil || ip.To16() == nil || ip.To4() != nil {
224+
return "", fmt.Errorf("not a valid IPv6 address: %q", ipStr)
225+
}
226+
mask := net.CIDRMask(64, 128)
227+
ipMasked := ip.Mask(mask)
228+
return (&net.IPNet{IP: ipMasked, Mask: mask}).String(), nil
137229
}
138230

231+
139232
// createV6OrDualNetwork creates a user-defined bridge network with IPv6 enabled,
140233
// and adds both subnets when ipFamily == "dual". Returns the gateway reported by inspect (may be nil for v6-only).
141234
func createV6OrDualNetwork(ociBin, name, subnetV4, subnetV6, ipFamily string) (net.IP, error) {

‎pkg/drivers/kic/oci/oci.go‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,26 @@ func CreateContainerNode(p CreateParams) error { //nolint to suppress cyclomatic
199199

200200
// For IPv6/dual clusters, enable forwarding inside the node container
201201
// (safe sysctl; avoid disable_ipv6 which may be blocked by Docker's safe list)
202-
if p.IPFamily == "ipv6" || p.IPFamily == "dual" {
203-
runArgs = append(runArgs, "--sysctl", "net.ipv6.conf.all.forwarding=1")
204-
}
205202

203+
// Ensure service rules apply to bridged traffic inside the node container.
204+
// Do both families; harmless if already set.
205+
runArgs = append(runArgs,
206+
"--sysctl", "net.ipv4.ip_forward=1",
207+
"--sysctl", "net.bridge.bridge-nf-call-iptables=1",
208+
// Allow kube-proxy/IPVS or iptables to program and accept IPv4 Service VIPs.
209+
"--sysctl", "net.ipv4.ip_nonlocal_bind=1",
210+
)
211+
// IPv6/dual clusters need IPv6 forwarding and IPv6 bridge netfilter, too.
212+
if p.IPFamily == "ipv6" || p.IPFamily == "dual" {
213+
runArgs = append(runArgs,
214+
"--sysctl", "net.ipv6.conf.all.forwarding=1",
215+
"--sysctl", "net.bridge.bridge-nf-call-ip6tables=1",
216+
// Allow kube-proxy/IPVS or iptables to program and accept Service VIPs.
217+
"--sysctl", "net.ipv4.ip_nonlocal_bind=1",
218+
// Same for IPv6 VIPs.
219+
"--sysctl", "net.ipv6.ip_nonlocal_bind=1",
220+
)
221+
}
206222
switch p.GPUs {
207223
case "all", "nvidia":
208224
runArgs = append(runArgs, "--gpus", "all", "--env", "NVIDIA_DRIVER_CAPABILITIES=all")

‎pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,10 @@ bootstrapTokens:
3636
nodeRegistration:
3737
criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
3838
name: "{{.NodeName}}"
39+
{{- if .NodeIP }}
3940
kubeletExtraArgs:
4041
node-ip: {{.NodeIP}}
42+
{{- end }}
4143
taints: []
4244
---
4345
apiVersion: kubeadm.k8s.io/v1beta1

0 commit comments

Comments
 (0)