Skip to content

Commit f5b88d9

Browse files
thomasferrandizthomasferrandiz
authored
Merge pull request #1866 from thomasferrandiz/simplify-trafficmanager-interface
Simplify trafficmanager interface
2 parents 94e2a27 + b76def1 commit f5b88d9

File tree

6 files changed

+179
-167
lines changed

6 files changed

+179
-167
lines changed

‎main.go

+41-132
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ import (
3131
"github.com/coreos/pkg/flagutil"
3232
"github.com/flannel-io/flannel/pkg/ip"
3333
"github.com/flannel-io/flannel/pkg/ipmatch"
34-
"github.com/flannel-io/flannel/pkg/lease"
3534
"github.com/flannel-io/flannel/pkg/subnet"
3635
etcd "github.com/flannel-io/flannel/pkg/subnet/etcd"
3736
"github.com/flannel-io/flannel/pkg/subnet/kube"
@@ -338,95 +337,56 @@ func main() {
338337

339338
//Create TrafficManager and instanciate it based on whether we use iptables or nftables
340339
trafficMngr := newTrafficManager()
340+
flannelIPv4Net := ip.IP4Net{}
341+
flannelIpv6Net := ip.IP6Net{}
342+
if config.EnableIPv4 {
343+
flannelIPv4Net, err = config.GetFlannelNetwork(&bn.Lease().Subnet)
344+
if err != nil {
345+
log.Error(err)
346+
cancel()
347+
wg.Wait()
348+
os.Exit(1)
349+
}
350+
}
351+
if config.EnableIPv6 {
352+
flannelIpv6Net, err = config.GetFlannelIPv6Network(&bn.Lease().IPv6Subnet)
353+
if err != nil {
354+
log.Error(err)
355+
cancel()
356+
wg.Wait()
357+
os.Exit(1)
358+
}
359+
}
341360
// Set up ipMasq if needed
342361
if opts.ipMasq {
343-
if config.EnableIPv4 {
344-
net, err := config.GetFlannelNetwork(&bn.Lease().Subnet)
345-
if err != nil {
346-
log.Error(err)
347-
cancel()
348-
wg.Wait()
349-
os.Exit(1)
350-
}
351-
if err = recycleIPTables(trafficMngr, net, bn.Lease()); err != nil {
352-
log.Errorf("Failed to recycle IPTables rules, %v", err)
353-
cancel()
354-
wg.Wait()
355-
os.Exit(1)
356-
}
357-
log.Infof("Setting up masking rules")
358-
trafficMngr.CreateIP4Chain("nat", "FLANNEL-POSTRTG")
359-
getRules := func() []trafficmngr.IPTablesRule {
360-
if config.HasNetworks() {
361-
return trafficMngr.MasqRules(config.Networks, bn.Lease())
362-
} else {
363-
return trafficMngr.MasqRules([]ip.IP4Net{config.Network}, bn.Lease())
364-
}
365-
}
366-
go trafficMngr.SetupAndEnsureIP4Tables(getRules, opts.iptablesResyncSeconds)
367-
368-
}
369-
if config.EnableIPv6 {
370-
ip6net, err := config.GetFlannelIPv6Network(&bn.Lease().IPv6Subnet)
371-
if err != nil {
372-
log.Error(err)
373-
cancel()
374-
wg.Wait()
375-
os.Exit(1)
376-
}
377-
if err = recycleIP6Tables(trafficMngr, ip6net, bn.Lease()); err != nil {
378-
log.Errorf("Failed to recycle IP6Tables rules, %v", err)
379-
cancel()
380-
wg.Wait()
381-
os.Exit(1)
382-
}
383-
log.Infof("Setting up masking ip6 rules")
384-
trafficMngr.CreateIP6Chain("nat", "FLANNEL-POSTRTG")
385-
getRules := func() []trafficmngr.IPTablesRule {
386-
if config.HasIPv6Networks() {
387-
return trafficMngr.MasqIP6Rules(config.IPv6Networks, bn.Lease())
388-
} else {
389-
return trafficMngr.MasqIP6Rules([]ip.IP6Net{config.IPv6Network}, bn.Lease())
390-
}
391-
}
392-
go trafficMngr.SetupAndEnsureIP6Tables(getRules, opts.iptablesResyncSeconds)
362+
prevNetworks := ReadCIDRsFromSubnetFile(opts.subnetFile, "FLANNEL_NETWORK")
363+
prevSubnet := ReadCIDRFromSubnetFile(opts.subnetFile, "FLANNEL_SUBNET")
364+
365+
prevIPv6Networks := ReadIP6CIDRsFromSubnetFile(opts.subnetFile, "FLANNEL_IPV6_NETWORK")
366+
prevIPv6Subnet := ReadIP6CIDRFromSubnetFile(opts.subnetFile, "FLANNEL_IPV6_SUBNET")
367+
368+
err = trafficMngr.SetupAndEnsureMasqRules(flannelIPv4Net, prevSubnet,
369+
prevNetworks,
370+
flannelIpv6Net, prevIPv6Subnet,
371+
prevIPv6Networks,
372+
bn.Lease(),
373+
opts.iptablesResyncSeconds)
374+
if err != nil {
375+
log.Errorf("Failed to setup masq rules, %v", err)
376+
cancel()
377+
wg.Wait()
378+
os.Exit(1)
393379
}
394380
}
395381

396382
// Always enables forwarding rules. This is needed for Docker versions >1.13 (https://docs.docker.com/engine/userguide/networking/default_network/container-communication/#container-communication-between-hosts)
397383
// In Docker 1.12 and earlier, the default FORWARD chain policy was ACCEPT.
398384
// In Docker 1.13 and later, Docker sets the default policy of the FORWARD chain to DROP.
399385
if opts.iptablesForwardRules {
400-
if config.EnableIPv4 {
401-
net, err := config.GetFlannelNetwork(&bn.Lease().Subnet)
402-
if err != nil {
403-
log.Error(err)
404-
cancel()
405-
wg.Wait()
406-
os.Exit(1)
407-
}
408-
log.Infof("Changing default FORWARD chain policy to ACCEPT")
409-
trafficMngr.CreateIP4Chain("filter", "FLANNEL-FWD")
410-
getRules := func() []trafficmngr.IPTablesRule {
411-
return trafficMngr.ForwardRules(net.String())
412-
}
413-
go trafficMngr.SetupAndEnsureIP4Tables(getRules, opts.iptablesResyncSeconds)
414-
}
415-
if config.EnableIPv6 {
416-
ip6net, err := config.GetFlannelIPv6Network(&bn.Lease().IPv6Subnet)
417-
if err != nil {
418-
log.Error(err)
419-
cancel()
420-
wg.Wait()
421-
os.Exit(1)
422-
}
423-
log.Infof("IPv6: Changing default FORWARD chain policy to ACCEPT")
424-
trafficMngr.CreateIP6Chain("filter", "FLANNEL-FWD")
425-
getRules := func() []trafficmngr.IPTablesRule {
426-
return trafficMngr.ForwardRules(ip6net.String())
427-
}
428-
go trafficMngr.SetupAndEnsureIP6Tables(getRules, opts.iptablesResyncSeconds)
429-
}
386+
trafficMngr.SetupAndEnsureForwardRules(
387+
flannelIPv4Net,
388+
flannelIpv6Net,
389+
opts.iptablesResyncSeconds)
430390
}
431391

432392
if err := sm.HandleSubnetFile(opts.subnetFile, config, opts.ipMasq, bn.Lease().Subnet, bn.Lease().IPv6Subnet, bn.MTU()); err != nil {
@@ -465,57 +425,6 @@ func main() {
465425
os.Exit(0)
466426
}
467427

468-
func recycleIPTables(tm trafficmngr.TrafficManager, nw ip.IP4Net, myLease *lease.Lease) error {
469-
prevNetworks := ReadCIDRsFromSubnetFile(opts.subnetFile, "FLANNEL_NETWORK")
470-
prevSubnet := ReadCIDRFromSubnetFile(opts.subnetFile, "FLANNEL_SUBNET")
471-
472-
//Find the cidr in FLANNEL_NETWORK which contains the podCIDR (i.e. FLANNEL_SUBNET) of this node
473-
prevNetwork := ip.IP4Net{}
474-
for _, net := range prevNetworks {
475-
if net.ContainsCIDR(&prevSubnet) {
476-
prevNetwork = net
477-
break
478-
}
479-
}
480-
// recycle iptables rules only when network configured or subnet leased is not equal to current one.
481-
if prevNetwork != nw && prevSubnet != myLease.Subnet {
482-
log.Infof("Current network or subnet (%v, %v) is not equal to previous one (%v, %v), trying to recycle old iptables rules", nw, myLease.Subnet, prevNetwork, prevSubnet)
483-
newLease := &lease.Lease{
484-
Subnet: prevSubnet,
485-
}
486-
if err := tm.DeleteIP4Tables(tm.MasqRules(prevNetworks, newLease)); err != nil {
487-
return err
488-
}
489-
}
490-
return nil
491-
}
492-
493-
func recycleIP6Tables(tm trafficmngr.TrafficManager, nw ip.IP6Net, myLease *lease.Lease) error {
494-
prevNetworks := ReadIP6CIDRsFromSubnetFile(opts.subnetFile, "FLANNEL_IPV6_NETWORK")
495-
prevSubnet := ReadIP6CIDRFromSubnetFile(opts.subnetFile, "FLANNEL_IPV6_SUBNET")
496-
497-
//Find the cidr in FLANNEL_IPV6_NETWORK which contains the podCIDR (i.e. FLANNEL_IPV6_SUBNET) of this node
498-
prevNetwork := ip.IP6Net{}
499-
for _, net := range prevNetworks {
500-
if net.ContainsCIDR(&prevSubnet) {
501-
prevNetwork = net
502-
break
503-
}
504-
}
505-
506-
// recycle iptables rules only when network configured or subnet leased is not equal to current one.
507-
if prevNetwork.String() != nw.String() && prevSubnet.String() != myLease.IPv6Subnet.String() {
508-
log.Infof("Current ipv6 network or subnet (%v, %v) is not equal to previous one (%v, %v), trying to recycle old ip6tables rules", nw, myLease.IPv6Subnet, prevNetwork, prevSubnet)
509-
lease := &lease.Lease{
510-
IPv6Subnet: prevSubnet,
511-
}
512-
if err := tm.DeleteIP6Tables(tm.MasqIP6Rules(prevNetworks, lease)); err != nil {
513-
return err
514-
}
515-
}
516-
return nil
517-
}
518-
519428
func shutdownHandler(ctx context.Context, sigs chan os.Signal, cancel context.CancelFunc) {
520429
// Wait for the context do be Done or for the signal to come in to shutdown.
521430
select {

‎pkg/subnet/config.go

+16
Original file line numberDiff line numberDiff line change
@@ -303,3 +303,19 @@ func (c *Config) HasIPv6Networks() bool {
303303
return false
304304
}
305305
}
306+
307+
func (c *Config) GetNetworks() []ip.IP4Net {
308+
if len(c.Networks) > 0 {
309+
return c.Networks
310+
} else {
311+
return []ip.IP4Net{c.Network}
312+
}
313+
}
314+
315+
func (c *Config) GeIPv6tNetworks() []ip.IP6Net {
316+
if len(c.Networks) > 0 {
317+
return c.IPv6Networks
318+
} else {
319+
return []ip.IP6Net{c.IPv6Network}
320+
}
321+
}

‎pkg/trafficmngr/iptables/iptables.go

+95-9
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,74 @@ type IPTablesManager struct{}
4444

4545
const kubeProxyMark string = "0x4000/0x4000"
4646

47-
func (iptm IPTablesManager) MasqRules(cluster_cidrs []ip.IP4Net, lease *lease.Lease) []trafficmngr.IPTablesRule {
47+
func (iptm IPTablesManager) SetupAndEnsureMasqRules(flannelIPv4Net, prevSubnet ip.IP4Net,
48+
prevNetworks []ip.IP4Net,
49+
flannelIPv6Net, prevIPv6Subnet ip.IP6Net,
50+
prevIPv6Networks []ip.IP6Net,
51+
currentlease *lease.Lease,
52+
resyncPeriod int) error {
53+
if flannelIPv4Net.String() != "" {
54+
//Find the cidr in FLANNEL_NETWORK which contains the podCIDR (i.e. FLANNEL_SUBNET) of this node
55+
prevNetwork := ip.IP4Net{}
56+
for _, net := range prevNetworks {
57+
if net.ContainsCIDR(&prevSubnet) {
58+
prevNetwork = net
59+
break
60+
}
61+
}
62+
// recycle iptables rules only when network configured or subnet leased is not equal to current one.
63+
if prevNetwork != flannelIPv4Net && prevSubnet != currentlease.Subnet {
64+
log.Infof("Current network or subnet (%v, %v) is not equal to previous one (%v, %v), trying to recycle old iptables rules",
65+
flannelIPv4Net, currentlease.Subnet, prevNetwork, prevSubnet)
66+
newLease := &lease.Lease{
67+
Subnet: prevSubnet,
68+
}
69+
if err := iptm.deleteIP4Tables(iptm.masqRules(prevNetworks, newLease)); err != nil {
70+
return err
71+
}
72+
}
73+
74+
log.Infof("Setting up masking rules")
75+
iptm.CreateIP4Chain("nat", "FLANNEL-POSTRTG")
76+
//Note: doesn't work for multiple networks but we disabled MultiClusterCIDR anyway
77+
getRules := func() []trafficmngr.IPTablesRule {
78+
return iptm.masqRules([]ip.IP4Net{flannelIPv4Net}, currentlease)
79+
}
80+
go iptm.setupAndEnsureIP4Tables(getRules, resyncPeriod)
81+
}
82+
if flannelIPv6Net.String() != "" {
83+
//Find the cidr in FLANNEL_IPV6_NETWORK which contains the podCIDR (i.e. FLANNEL_IPV6_SUBNET) of this node
84+
prevIPv6Network := ip.IP6Net{}
85+
for _, net := range prevIPv6Networks {
86+
if net.ContainsCIDR(&prevIPv6Subnet) {
87+
prevIPv6Network = net
88+
break
89+
}
90+
}
91+
// recycle iptables rules only when network configured or subnet leased is not equal to current one.
92+
if prevIPv6Network != flannelIPv6Net && prevIPv6Subnet != currentlease.IPv6Subnet {
93+
log.Infof("Current network or subnet (%v, %v) is not equal to previous one (%v, %v), trying to recycle old iptables rules",
94+
flannelIPv6Net, currentlease.IPv6Subnet, prevIPv6Network, prevIPv6Subnet)
95+
newLease := &lease.Lease{
96+
IPv6Subnet: prevIPv6Subnet,
97+
}
98+
if err := iptm.deleteIP6Tables(iptm.masqIP6Rules(prevIPv6Networks, newLease)); err != nil {
99+
return err
100+
}
101+
}
102+
103+
log.Infof("Setting up masking rules for IPv6")
104+
iptm.CreateIP6Chain("nat", "FLANNEL-POSTRTG")
105+
//Note: doesn't work for multiple networks but we disabled MultiClusterCIDR anyway
106+
getRules := func() []trafficmngr.IPTablesRule {
107+
return iptm.masqIP6Rules([]ip.IP6Net{flannelIPv6Net}, currentlease)
108+
}
109+
go iptm.setupAndEnsureIP6Tables(getRules, resyncPeriod)
110+
}
111+
return nil
112+
}
113+
114+
func (iptm IPTablesManager) masqRules(cluster_cidrs []ip.IP4Net, lease *lease.Lease) []trafficmngr.IPTablesRule {
48115
pod_cidr := lease.Subnet.String()
49116
ipt, err := iptables.New()
50117
supports_random_fully := false
@@ -90,7 +157,7 @@ func (iptm IPTablesManager) MasqRules(cluster_cidrs []ip.IP4Net, lease *lease.Le
90157
return rules
91158
}
92159

93-
func (iptm IPTablesManager) MasqIP6Rules(cluster_cidrs []ip.IP6Net, lease *lease.Lease) []trafficmngr.IPTablesRule {
160+
func (iptm IPTablesManager) masqIP6Rules(cluster_cidrs []ip.IP6Net, lease *lease.Lease) []trafficmngr.IPTablesRule {
94161
pod_cidr := lease.IPv6Subnet.String()
95162
ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
96163
supports_random_fully := false
@@ -141,7 +208,26 @@ func (iptm IPTablesManager) MasqIP6Rules(cluster_cidrs []ip.IP6Net, lease *lease
141208
return rules
142209
}
143210

144-
func (iptm IPTablesManager) ForwardRules(flannelNetwork string) []trafficmngr.IPTablesRule {
211+
func (iptm IPTablesManager) SetupAndEnsureForwardRules(flannelIPv4Network ip.IP4Net, flannelIPv6Network ip.IP6Net, resyncPeriod int) {
212+
if flannelIPv4Network.String() != "" {
213+
log.Infof("Changing default FORWARD chain policy to ACCEPT")
214+
iptm.CreateIP4Chain("filter", "FLANNEL-FWD")
215+
getRules := func() []trafficmngr.IPTablesRule {
216+
return iptm.forwardRules(flannelIPv4Network.String())
217+
}
218+
go iptm.setupAndEnsureIP4Tables(getRules, resyncPeriod)
219+
}
220+
if flannelIPv6Network.String() != "" {
221+
log.Infof("IPv6: Changing default FORWARD chain policy to ACCEPT")
222+
iptm.CreateIP6Chain("filter", "FLANNEL-FWD")
223+
getRules := func() []trafficmngr.IPTablesRule {
224+
return iptm.forwardRules(flannelIPv6Network.String())
225+
}
226+
go iptm.setupAndEnsureIP6Tables(getRules, resyncPeriod)
227+
}
228+
}
229+
230+
func (iptm IPTablesManager) forwardRules(flannelNetwork string) []trafficmngr.IPTablesRule {
145231
return []trafficmngr.IPTablesRule{
146232
// This rule ensure that the flannel iptables rules are executed before other rules on the node
147233
{Table: "filter", Action: "-A", Chain: "FORWARD", Rulespec: []string{"-m", "comment", "--comment", "flanneld forward", "-j", "FLANNEL-FWD"}},
@@ -281,7 +367,7 @@ func ipTablesBootstrap(ipt IPTables, iptRestore IPTablesRestore, rules []traffic
281367
return nil
282368
}
283369

284-
func (iptm IPTablesManager) SetupAndEnsureIP4Tables(getRules func() []trafficmngr.IPTablesRule, resyncPeriod int) {
370+
func (iptm IPTablesManager) setupAndEnsureIP4Tables(getRules func() []trafficmngr.IPTablesRule, resyncPeriod int) {
285371
rules := getRules()
286372
log.Infof("generated %d rules", len(rules))
287373
ipt, err := iptables.New()
@@ -320,7 +406,7 @@ func (iptm IPTablesManager) SetupAndEnsureIP4Tables(getRules func() []trafficmng
320406
}
321407
}
322408

323-
func (iptm IPTablesManager) SetupAndEnsureIP6Tables(getRules func() []trafficmngr.IPTablesRule, resyncPeriod int) {
409+
func (iptm IPTablesManager) setupAndEnsureIP6Tables(getRules func() []trafficmngr.IPTablesRule, resyncPeriod int) {
324410
rules := getRules()
325411
ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
326412
if err != nil {
@@ -358,8 +444,8 @@ func (iptm IPTablesManager) SetupAndEnsureIP6Tables(getRules func() []trafficmng
358444
}
359445
}
360446

361-
// DeleteIP4Tables delete specified iptables rules
362-
func (iptm IPTablesManager) DeleteIP4Tables(rules []trafficmngr.IPTablesRule) error {
447+
// deleteIP4Tables delete specified iptables rules
448+
func (iptm IPTablesManager) deleteIP4Tables(rules []trafficmngr.IPTablesRule) error {
363449
ipt, err := iptables.New()
364450
if err != nil {
365451
// if we can't find iptables, give up and return
@@ -380,8 +466,8 @@ func (iptm IPTablesManager) DeleteIP4Tables(rules []trafficmngr.IPTablesRule) er
380466
return nil
381467
}
382468

383-
// DeleteIP6Tables delete specified iptables rules
384-
func (iptm IPTablesManager) DeleteIP6Tables(rules []trafficmngr.IPTablesRule) error {
469+
// deleteIP6Tables delete specified iptables rules
470+
func (iptm IPTablesManager) deleteIP6Tables(rules []trafficmngr.IPTablesRule) error {
385471
ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
386472
if err != nil {
387473
// if we can't find iptables, give up and return

‎pkg/trafficmngr/iptables/iptables_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ func TestDeleteRules(t *testing.T) {
121121
ipt := &MockIPTables{t: t}
122122
iptr := &MockIPTablesRestore{t: t}
123123
iptm := IPTablesManager{}
124-
baseRules := iptm.MasqRules([]ip.IP4Net{{
124+
baseRules := iptm.masqRules([]ip.IP4Net{{
125125
IP: ip.MustParseIP4("10.0.1.0"),
126126
PrefixLen: 16,
127127
}}, testingLease())

0 commit comments

Comments
 (0)