Skip to content

Commit ad69d5e

Browse files
GreatLazyManGreatLazyMan
committed
title: Add ipsec tunnel mode to support cross clusters and elastic ip
Signed-off-by: GreatLazyMan <[email protected]>
1 parent 04938bf commit ad69d5e

File tree

34 files changed

+1949
-94
lines changed

34 files changed

+1949
-94
lines changed

‎cluster/images/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ ARG BINARY
44

55
RUN apk add --no-cache ca-certificates
66
RUN apk update && apk upgrade
7-
RUN apk add ip6tables iptables curl tcpdump busybox-extras
7+
RUN apk add ip6tables iptables ipset curl tcpdump busybox-extras
88

99
COPY ${BINARY} /bin/${BINARY}

‎cmd/clusterlink/elector/app/elector.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,10 +126,11 @@ func run(ctx context.Context, opts *options.Options) error {
126126
err := elector.EnsureGateWayRole()
127127
if err != nil {
128128
klog.Errorf("set gateway role failure: %v, retry after 10 sec.", err)
129-
time.Sleep(10 * time.Second)
129+
time.Sleep(3 * time.Second)
130130
} else {
131-
klog.V(4).Info("ensure gateway role success, recheck after 60 sec.")
132-
time.Sleep(60 * time.Second)
131+
timeToRecheck := 3 * time.Second
132+
klog.V(4).Infof("ensure gateway role success, recheck after %d sec.", int(timeToRecheck))
133+
time.Sleep(timeToRecheck)
133134
}
134135
}
135136
}

‎deploy/crds/kosmos.io_clusternodes.yaml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ spec:
4545
properties:
4646
clusterName:
4747
type: string
48+
elasticip:
49+
type: string
4850
interfaceName:
4951
type: string
5052
ip:
@@ -63,11 +65,13 @@ spec:
6365
type: array
6466
type: object
6567
status:
68+
properties:
69+
nodeStatus:
70+
type: string
6671
type: object
6772
required:
6873
- spec
6974
type: object
7075
served: true
7176
storage: true
72-
subresources:
73-
status: {}
77+
subresources: {}

‎deploy/crds/kosmos.io_clusters.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ spec:
5858
- ip
5959
- ip6
6060
type: object
61+
clusterpodCIDRs:
62+
items:
63+
type: string
64+
type: array
6165
cni:
6266
default: calico
6367
type: string
@@ -107,9 +111,17 @@ spec:
107111
- nodeName
108112
type: object
109113
type: array
114+
nodeElasticIPMap:
115+
additionalProperties:
116+
type: string
117+
description: NodeElasticIPMap presents mapping between nodename
118+
in kubernetes and elasticIP
119+
type: object
110120
useIPPool:
111121
default: false
112122
type: boolean
123+
useexternalapiserver:
124+
type: boolean
113125
type: object
114126
clusterTreeOptions:
115127
properties:

‎deploy/crds/kosmos.io_nodeconfigs.yaml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,18 @@ spec:
9292
- mac
9393
type: object
9494
type: array
95+
ipsetsavoidmasq:
96+
items:
97+
properties:
98+
cidr:
99+
type: string
100+
name:
101+
type: string
102+
required:
103+
- cidr
104+
- name
105+
type: object
106+
type: array
95107
iptables:
96108
items:
97109
properties:
@@ -122,6 +134,52 @@ spec:
122134
- gw
123135
type: object
124136
type: array
137+
xfrmpolicies:
138+
items:
139+
properties:
140+
dir:
141+
type: integer
142+
leftip:
143+
type: string
144+
leftnet:
145+
type: string
146+
reqid:
147+
type: integer
148+
rightip:
149+
type: string
150+
rightnet:
151+
type: string
152+
required:
153+
- dir
154+
- leftip
155+
- leftnet
156+
- reqid
157+
- rightip
158+
- rightnet
159+
type: object
160+
type: array
161+
xfrmstates:
162+
items:
163+
properties:
164+
PSK:
165+
type: string
166+
leftip:
167+
type: string
168+
reqid:
169+
type: integer
170+
rightip:
171+
type: string
172+
spi:
173+
format: int32
174+
type: integer
175+
required:
176+
- PSK
177+
- leftip
178+
- reqid
179+
- rightip
180+
- spi
181+
type: object
182+
type: array
125183
type: object
126184
status:
127185
properties:

‎pkg/apis/kosmos/v1alpha1/cluster_types.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,14 @@ type ClusterLinkOptions struct {
9494

9595
// +optional
9696
AutodetectionMethod string `json:"autodetectionMethod,omitempty"`
97+
98+
// NodeElasticIPMap presents mapping between nodename in kubernetes and elasticIP
99+
// +optional
100+
NodeElasticIPMap map[string]string `json:"nodeElasticIPMap,omitempty"`
101+
// +optional
102+
ClusterPodCIDRs []string `json:"clusterpodCIDRs,omitempty"`
103+
// +optional
104+
UseExternalApiserver bool `json:"useexternalapiserver,omitempty"`
97105
}
98106

99107
type ClusterTreeOptions struct {

‎pkg/apis/kosmos/v1alpha1/clusternode_types.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88

99
// +genclient
1010
// +genclient:nonNamespaced
11-
// +kubebuilder:subresource:status
1211
// +kubebuilder:resource:scope="Cluster"
1312
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
1413
// +kubebuilder:printcolumn:name="ROLES",type=string,JSONPath=`.spec.roles`
@@ -33,6 +32,8 @@ type ClusterNodeSpec struct {
3332
// +optional
3433
IP string `json:"ip,omitempty"`
3534
// +optional
35+
ElasticIP string `json:"elasticip,omitempty"`
36+
// +optional
3637
IP6 string `json:"ip6,omitempty"`
3738
// +optional
3839
Roles []Role `json:"roles,omitempty"`
@@ -41,6 +42,8 @@ type ClusterNodeSpec struct {
4142
}
4243

4344
type ClusterNodeStatus struct {
45+
// +optional
46+
NodeStatus string `json:"nodeStatus,omitempty"`
4447
}
4548

4649
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

‎pkg/apis/kosmos/v1alpha1/constants.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,17 @@ type DeviceType string
2828
const (
2929
VxlanDevice DeviceType = "vxlan"
3030
)
31+
32+
const (
33+
DefaultPSK string = "bfd6224354977084568832b811226b3d6cff6685"
34+
DefaultPSKPreStr = "WelcometoKosmos"
35+
DefaultReqID int = 336
36+
)
37+
38+
type IPSECDirection int
39+
40+
const (
41+
IPSECIn IPSECDirection = 0
42+
IPSECOut IPSECDirection = 1
43+
IPSECFwd IPSECDirection = 2
44+
)

‎pkg/apis/kosmos/v1alpha1/nodeconfig_types.go‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,14 @@ type NodeConfig struct {
2121
}
2222

2323
type NodeConfigSpec struct {
24-
Devices []Device `json:"devices,omitempty"`
25-
Routes []Route `json:"routes,omitempty"`
26-
Iptables []Iptables `json:"iptables,omitempty"`
27-
Fdbs []Fdb `json:"fdbs,omitempty"`
28-
Arps []Arp `json:"arps,omitempty"`
24+
Devices []Device `json:"devices,omitempty"`
25+
Routes []Route `json:"routes,omitempty"`
26+
Iptables []Iptables `json:"iptables,omitempty"`
27+
Fdbs []Fdb `json:"fdbs,omitempty"`
28+
Arps []Arp `json:"arps,omitempty"`
29+
XfrmPolicies []XfrmPolicy `json:"xfrmpolicies,omitempty"`
30+
XfrmStates []XfrmState `json:"xfrmstates,omitempty"`
31+
IPsetsAvoidMasqs []IPset `json:"ipsetsavoidmasq,omitempty"`
2932
}
3033

3134
type NodeConfigStatus struct {
@@ -101,6 +104,50 @@ func (a *Arp) Compare(v Arp) bool {
101104
a.Dev == v.Dev
102105
}
103106

107+
type XfrmPolicy struct {
108+
LeftIP string `json:"leftip"`
109+
LeftNet string `json:"leftnet"`
110+
RightIP string `json:"rightip"`
111+
RightNet string `json:"rightnet"`
112+
ReqID int `json:"reqid"`
113+
Dir int `json:"dir"`
114+
}
115+
116+
func (a *XfrmPolicy) Compare(v XfrmPolicy) bool {
117+
return a.LeftIP == v.LeftIP &&
118+
a.LeftNet == v.LeftNet &&
119+
a.RightNet == v.RightNet &&
120+
a.RightIP == v.RightIP &&
121+
a.ReqID == v.ReqID &&
122+
a.Dir == v.Dir
123+
}
124+
125+
type XfrmState struct {
126+
LeftIP string `json:"leftip"`
127+
RightIP string `json:"rightip"`
128+
ReqID int `json:"reqid"`
129+
SPI uint32 `json:"spi"`
130+
PSK string `json:"PSK"`
131+
}
132+
133+
func (a *XfrmState) Compare(v XfrmState) bool {
134+
return a.LeftIP == v.LeftIP &&
135+
a.RightIP == v.RightIP &&
136+
a.ReqID == v.ReqID &&
137+
a.PSK == v.PSK &&
138+
a.SPI == v.SPI
139+
}
140+
141+
type IPset struct {
142+
CIDR string `json:"cidr"`
143+
Name string `json:"name"`
144+
}
145+
146+
func (a *IPset) Compare(v IPset) bool {
147+
return a.CIDR == v.CIDR &&
148+
a.Name == v.Name
149+
}
150+
104151
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
105152

106153
type NodeConfigList struct {

‎pkg/clusterlink/agent-manager/network-manager/network_manager.go‎

Lines changed: 67 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,29 @@ func (e *NetworkManager) Diff(oldConfig, newConfig *clusterlinkv1alpha1.NodeConf
112112
createConfig.Routes = createRecord
113113
isSame = false
114114
}
115+
// ipsec:
116+
if flag, deleteRecord, createRecord := compareFunc(oldConfig.XfrmPolicies, newConfig.XfrmPolicies, func(a, b clusterlinkv1alpha1.XfrmPolicy) bool {
117+
return a.Compare(b)
118+
}); !flag {
119+
deleteConfig.XfrmPolicies = deleteRecord
120+
createConfig.XfrmPolicies = createRecord
121+
isSame = false
122+
}
123+
if flag, deleteRecord, createRecord := compareFunc(oldConfig.XfrmStates, newConfig.XfrmStates, func(a, b clusterlinkv1alpha1.XfrmState) bool {
124+
return a.Compare(b)
125+
}); !flag {
126+
deleteConfig.XfrmStates = deleteRecord
127+
createConfig.XfrmStates = createRecord
128+
isSame = false
129+
}
130+
//ipset
131+
if flag, deleteRecord, createRecord := compareFunc(oldConfig.IPsetsAvoidMasqs, newConfig.IPsetsAvoidMasqs, func(a, b clusterlinkv1alpha1.IPset) bool {
132+
return a.Compare(b)
133+
}); !flag {
134+
deleteConfig.IPsetsAvoidMasqs = deleteRecord
135+
createConfig.IPsetsAvoidMasqs = createRecord
136+
isSame = false
137+
}
115138
// iptables:
116139
if flag, deleteRecord, createRecord := compareFunc(oldConfig.Iptables, newConfig.Iptables, func(a, b clusterlinkv1alpha1.Iptables) bool {
117140
return a.Compare(b)
@@ -188,6 +211,24 @@ func (e *NetworkManager) WriteSys(configDiff *ConfigDiff) error {
188211
errs = errors.Wrap(err, fmt.Sprint(errs))
189212
}
190213
}
214+
if config.XfrmPolicies != nil {
215+
if err := e.NetworkInterface.DeleteXfrmPolicies(config.XfrmPolicies); err != nil {
216+
klog.Warning(err)
217+
errs = errors.Wrap(err, fmt.Sprint(errs))
218+
}
219+
}
220+
if config.XfrmStates != nil {
221+
if err := e.NetworkInterface.DeleteXfrmStates(config.XfrmStates); err != nil {
222+
klog.Warning(err)
223+
errs = errors.Wrap(err, fmt.Sprint(errs))
224+
}
225+
}
226+
if config.IPsetsAvoidMasqs != nil {
227+
if err := e.NetworkInterface.DeleteIPsetsAvoidMasq(config.IPsetsAvoidMasqs); err != nil {
228+
klog.Warning(err)
229+
errs = errors.Wrap(err, fmt.Sprint(errs))
230+
}
231+
}
191232
}
192233

193234
if configDiff.createConfig != nil {
@@ -223,6 +264,24 @@ func (e *NetworkManager) WriteSys(configDiff *ConfigDiff) error {
223264
errs = errors.Wrap(err, fmt.Sprint(errs))
224265
}
225266
}
267+
if config.XfrmPolicies != nil {
268+
if err := e.NetworkInterface.AddXfrmPolicies(config.XfrmPolicies); err != nil {
269+
klog.Warning(err)
270+
errs = errors.Wrap(err, fmt.Sprint(errs))
271+
}
272+
}
273+
if config.XfrmStates != nil {
274+
if err := e.NetworkInterface.AddXfrmStates(config.XfrmStates); err != nil {
275+
klog.Warning(err)
276+
errs = errors.Wrap(err, fmt.Sprint(errs))
277+
}
278+
}
279+
if config.IPsetsAvoidMasqs != nil {
280+
if err := e.NetworkInterface.AddIPsetsAvoidMasq(config.IPsetsAvoidMasqs); err != nil {
281+
klog.Warning(err)
282+
errs = errors.Wrap(err, fmt.Sprint(errs))
283+
}
284+
}
226285
}
227286

228287
return errs
@@ -254,11 +313,14 @@ func (e *NetworkManager) UpdateFromChecker() NodeConfigSyncStatus {
254313
}
255314

256315
func printNodeConfig(data *clusterlinkv1alpha1.NodeConfigSpec) {
257-
klog.Infof("device: ", data.Devices)
258-
klog.Infof("Arps: ", data.Arps)
259-
klog.Infof("Fdbs: ", data.Fdbs)
260-
klog.Infof("Iptables: ", data.Iptables)
261-
klog.Infof("Routes: ", data.Routes)
316+
klog.Infof("device: %v", data.Devices)
317+
klog.Infof("Arps: %v", data.Arps)
318+
klog.Infof("Fdbs: %v", data.Fdbs)
319+
klog.Infof("Iptables: %v", data.Iptables)
320+
klog.Infof("Routes: %v", data.Routes)
321+
klog.Infof("XfrmPolicys: %v", data.XfrmPolicies)
322+
klog.Infof("XfrmStates: %v", data.XfrmStates)
323+
klog.Infof("IPsetsAvoidMasqs: %v", data.IPsetsAvoidMasqs)
262324
}
263325

264326
func (e *NetworkManager) UpdateSync() NodeConfigSyncStatus {

0 commit comments

Comments
 (0)