Skip to content

Commit 2646038

Browse files
DockToFutureDockToFuture
committed
add dual-stack support for node-cache
1 parent fe97ec9 commit 2646038

File tree

2 files changed

+125
-70
lines changed

2 files changed

+125
-70
lines changed

‎cmd/node-cache/app/cache_app.go‎

Lines changed: 125 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"k8s.io/dns/cmd/kube-dns/app/options"
2626
"k8s.io/dns/pkg/dns/config"
2727
"k8s.io/dns/pkg/netif"
28+
"k8s.io/kubernetes/pkg/util/iptables"
2829
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
2930
utilexec "k8s.io/utils/exec"
3031
utilnet "k8s.io/utils/net"
@@ -62,13 +63,15 @@ type iptablesRule struct {
6263

6364
// CacheApp contains all the config required to run node-cache.
6465
type CacheApp struct {
65-
iptables utiliptables.Interface
66-
iptablesRules []iptablesRule
67-
params *ConfigParams
68-
netifHandle *netif.NetifManager
69-
kubednsConfig *options.KubeDNSConfig
70-
exitChan chan struct{} // Channel to terminate background goroutines
71-
clusterDNSIP net.IP
66+
iptablesV4 utiliptables.Interface
67+
iptablesV6 utiliptables.Interface
68+
iptablesRulesV4 []iptablesRule
69+
iptablesRulesV6 []iptablesRule
70+
params *ConfigParams
71+
netifHandle *netif.NetifManager
72+
kubednsConfig *options.KubeDNSConfig
73+
exitChan chan struct{} // Channel to terminate background goroutines
74+
clusterDNSIP net.IP
7275
}
7376

7477
func isLockedErr(err error) bool {
@@ -98,61 +101,86 @@ func (c *CacheApp) Init() {
98101
c.params.SetupIptables = setupIptables
99102
}
100103

101-
// isIPv6 return if the node-cache is working in IPv6 mode
102-
// LocalIPs are guaranteed to have the same family
103-
func (c *CacheApp) isIPv6() bool {
104-
if len(c.params.LocalIPs) > 0 {
105-
return utilnet.IsIPv6(c.params.LocalIPs[0])
106-
}
107-
return false
108-
}
109-
110104
func (c *CacheApp) initIptables() {
111105
// using the localIPStr param since we need ip strings here
112106
for _, localIP := range strings.Split(c.params.LocalIPStr, ",") {
113-
c.iptablesRules = append(c.iptablesRules, []iptablesRule{
114-
// Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
115-
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP,
116-
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
117-
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP,
118-
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
119-
// There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking,
120-
// need these additional filter table rules.
121-
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP,
122-
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
123-
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP,
124-
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
125-
// Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
126-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
127-
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
128-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
129-
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
130-
// Additional filter table rules for traffic frpm localIp:localPort
131-
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
132-
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
133-
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
134-
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
135-
// Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods
136-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
137-
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
138-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP,
139-
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
140-
// skip connection tracking for healthcheck requests generated by liveness probe to health plugin
141-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
142-
"--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
143-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
144-
"--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
145-
}...)
107+
if utilnet.IsIPv6(net.ParseIP(localIP)) {
108+
c.iptablesRulesV6 = append(c.iptablesRulesV6, []iptablesRule{
109+
// Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
110+
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP,
111+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
112+
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP,
113+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
114+
// There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking,
115+
// need these additional filter table rules.
116+
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP,
117+
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
118+
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP,
119+
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
120+
// Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
121+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
122+
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
123+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
124+
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
125+
// Additional filter table rules for traffic frpm localIp:localPort
126+
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
127+
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
128+
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
129+
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
130+
// Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods
131+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
132+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
133+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP,
134+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
135+
// skip connection tracking for healthcheck requests generated by liveness probe to health plugin
136+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
137+
"--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
138+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
139+
"--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
140+
}...)
141+
} else {
142+
c.iptablesRulesV4 = append(c.iptablesRulesV4, []iptablesRule{
143+
// Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
144+
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP,
145+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
146+
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP,
147+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
148+
// There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking,
149+
// need these additional filter table rules.
150+
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP,
151+
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
152+
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP,
153+
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
154+
// Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
155+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
156+
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
157+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
158+
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
159+
// Additional filter table rules for traffic frpm localIp:localPort
160+
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
161+
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
162+
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
163+
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
164+
// Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods
165+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
166+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
167+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP,
168+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
169+
// skip connection tracking for healthcheck requests generated by liveness probe to health plugin
170+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
171+
"--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
172+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
173+
"--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
174+
}...)
175+
}
176+
146177
}
147-
c.iptables = newIPTables(c.isIPv6())
178+
c.iptablesV4 = newIPTables(iptables.ProtocolIPv4)
179+
c.iptablesV6 = newIPTables(iptables.ProtocolIPv6)
148180
}
149181

150-
func newIPTables(isIPv6 bool) utiliptables.Interface {
182+
func newIPTables(protocol iptables.Protocol) utiliptables.Interface {
151183
execer := utilexec.New()
152-
protocol := utiliptables.ProtocolIPv4
153-
if isIPv6 {
154-
protocol = utiliptables.ProtocolIPv6
155-
}
156184
return utiliptables.New(execer, protocol)
157185
}
158186

@@ -180,32 +208,66 @@ func (c *CacheApp) TeardownNetworking() error {
180208
err = c.netifHandle.RemoveDummyDevice(c.params.InterfaceName)
181209
}
182210
if c.params.SetupIptables {
183-
for _, rule := range c.iptablesRules {
211+
for _, rule := range c.iptablesRulesV4 {
212+
exists := true
213+
for exists == true {
214+
// check in a loop in case the same rule got added multiple times.
215+
err = c.iptablesV4.DeleteRule(rule.table, rule.chain, rule.args...)
216+
if err != nil {
217+
clog.Errorf("Failed deleting iptablesV4 rule %v, error - %v", rule, err)
218+
handleIPTablesError(err)
219+
}
220+
exists, err = c.iptablesV4.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
221+
if err != nil {
222+
clog.Errorf("Failed checking iptablesV4 rule after deletion, rule - %v, error - %v", rule, err)
223+
handleIPTablesError(err)
224+
}
225+
}
226+
// Delete the rule one last time since EnsureRule creates the rule if it doesn't exist
227+
err = c.iptablesV4.DeleteRule(rule.table, rule.chain, rule.args...)
228+
}
229+
for _, rule := range c.iptablesRulesV6 {
184230
exists := true
185231
for exists == true {
186232
// check in a loop in case the same rule got added multiple times.
187-
err = c.iptables.DeleteRule(rule.table, rule.chain, rule.args...)
233+
err = c.iptablesV6.DeleteRule(rule.table, rule.chain, rule.args...)
188234
if err != nil {
189-
clog.Errorf("Failed deleting iptables rule %v, error - %v", rule, err)
235+
clog.Errorf("Failed deleting iptablesV6 rule %v, error - %v", rule, err)
190236
handleIPTablesError(err)
191237
}
192-
exists, err = c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
238+
exists, err = c.iptablesV6.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
193239
if err != nil {
194-
clog.Errorf("Failed checking iptables rule after deletion, rule - %v, error - %v", rule, err)
240+
clog.Errorf("Failed checking iptablesV6 rule after deletion, rule - %v, error - %v", rule, err)
195241
handleIPTablesError(err)
196242
}
197243
}
198244
// Delete the rule one last time since EnsureRule creates the rule if it doesn't exist
199-
err = c.iptables.DeleteRule(rule.table, rule.chain, rule.args...)
245+
err = c.iptablesV6.DeleteRule(rule.table, rule.chain, rule.args...)
200246
}
201247
}
202248
return err
203249
}
204250

205251
func (c *CacheApp) setupNetworking() {
206252
if c.params.SetupIptables {
207-
for _, rule := range c.iptablesRules {
208-
exists, err := c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
253+
for _, rule := range c.iptablesRulesV4 {
254+
exists, err := c.iptablesV4.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
255+
switch {
256+
case exists:
257+
// debug messages can be printed by including "debug" plugin in coreFile.
258+
clog.Debugf("iptables rule %v for nodelocaldns already exists", rule)
259+
continue
260+
case err == nil:
261+
clog.Infof("Added back nodelocaldns rule - %v", rule)
262+
continue
263+
default:
264+
// iptables check/rule add failed with error since control reached here.
265+
clog.Errorf("Error checking/adding iptables rule %v, error - %v", rule, err)
266+
handleIPTablesError(err)
267+
}
268+
}
269+
for _, rule := range c.iptablesRulesV6 {
270+
exists, err := c.iptablesV6.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
209271
switch {
210272
case exists:
211273
// debug messages can be printed by including "debug" plugin in coreFile.

‎cmd/node-cache/main.go‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ import (
2525

2626
corednsmain "github.com/coredns/coredns/coremain"
2727
clog "github.com/coredns/coredns/plugin/pkg/log"
28-
utilnet "k8s.io/utils/net"
2928

3029
"github.com/coredns/caddy"
3130
// blank imports to make sure the plugin code is pulled in from vendor when building node-cache image
@@ -100,12 +99,6 @@ func parseAndValidateFlags() (*app.ConfigParams, error) {
10099
params.LocalIPs = append(params.LocalIPs, newIP)
101100
}
102101

103-
// validate all the IPs have the same IP family
104-
for _, ip := range params.LocalIPs {
105-
if utilnet.IsIPv6(params.LocalIPs[0]) != utilnet.IsIPv6(ip) {
106-
return params, fmt.Errorf("unexpected IP Family for localIP - %q, want IPv6=%v", ip, utilnet.IsIPv6(params.LocalIPs[0]))
107-
}
108-
}
109102
// lookup specified dns port
110103
f := flag.Lookup("dns.port")
111104
if f == nil {

0 commit comments

Comments
 (0)