Skip to content

Commit 70561f0

Browse files
DockToFutureDockToFuture
committed
add dual-stack support for node-cache
1 parent 1d8f1a6 commit 70561f0

File tree

2 files changed

+124
-74
lines changed

2 files changed

+124
-74
lines changed

‎cmd/node-cache/app/cache_app.go‎

Lines changed: 124 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,10 @@ import (
2222

2323
"github.com/coredns/coredns/coremain"
2424
clog "github.com/coredns/coredns/plugin/pkg/log"
25-
2625
"k8s.io/dns/cmd/kube-dns/app/options"
2726
"k8s.io/dns/pkg/dns/config"
2827
"k8s.io/dns/pkg/netif"
28+
2929
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
3030
utilnet "k8s.io/utils/net"
3131
)
@@ -63,14 +63,16 @@ type iptablesRule struct {
6363

6464
// CacheApp contains all the config required to run node-cache.
6565
type CacheApp struct {
66-
iptables utiliptables.Interface
67-
iptablesRules []iptablesRule
68-
params *ConfigParams
69-
netifHandle *netif.NetifManager
70-
kubednsConfig *options.KubeDNSConfig
71-
exitChan chan struct{} // Channel to terminate background goroutines
72-
clusterDNSIP net.IP
73-
selfProcess *os.Process
66+
iptablesV4 utiliptables.Interface
67+
iptablesV6 utiliptables.Interface
68+
iptablesRulesV4 []iptablesRule
69+
iptablesRulesV6 []iptablesRule
70+
params *ConfigParams
71+
netifHandle *netif.NetifManager
72+
kubednsConfig *options.KubeDNSConfig
73+
exitChan chan struct{} // Channel to terminate background goroutines
74+
clusterDNSIP net.IP
75+
selfProcess *os.Process
7476
}
7577

7678
func isLockedErr(err error) bool {
@@ -100,61 +102,82 @@ func (c *CacheApp) Init() {
100102
c.params.SetupIptables = setupIptables
101103
}
102104

103-
// isIPv6 return if the node-cache is working in IPv6 mode
104-
// LocalIPs are guaranteed to have the same family
105-
func (c *CacheApp) isIPv6() bool {
106-
if len(c.params.LocalIPs) > 0 {
107-
return utilnet.IsIPv6(c.params.LocalIPs[0])
108-
}
109-
return false
110-
}
111-
112105
func (c *CacheApp) initIptables() {
113106
// using the localIPStr param since we need ip strings here
114107
for _, localIP := range strings.Split(c.params.LocalIPStr, ",") {
115-
c.iptablesRules = append(c.iptablesRules, []iptablesRule{
116-
// Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
117-
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP,
118-
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
119-
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP,
120-
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
121-
// There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking,
122-
// need these additional filter table rules.
123-
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP,
124-
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
125-
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP,
126-
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
127-
// Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
128-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
129-
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
130-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
131-
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
132-
// Additional filter table rules for traffic frpm localIp:localPort
133-
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
134-
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
135-
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
136-
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
137-
// Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods
138-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
139-
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
140-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP,
141-
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
142-
// skip connection tracking for healthcheck requests generated by liveness probe to health plugin
143-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
144-
"--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
145-
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
146-
"--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
147-
}...)
148-
}
149-
c.iptables = newIPTables(c.isIPv6())
150-
}
108+
if utilnet.IsIPv6(net.ParseIP(localIP)) {
109+
c.iptablesRulesV6 = append(c.iptablesRulesV6, []iptablesRule{
110+
// Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
111+
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP,
112+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
113+
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP,
114+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
115+
// There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking,
116+
// need these additional filter table rules.
117+
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP,
118+
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
119+
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP,
120+
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
121+
// Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
122+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
123+
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
124+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
125+
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
126+
// Additional filter table rules for traffic frpm localIp:localPort
127+
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
128+
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
129+
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
130+
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
131+
// Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods
132+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
133+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
134+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP,
135+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
136+
// skip connection tracking for healthcheck requests generated by liveness probe to health plugin
137+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
138+
"--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
139+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
140+
"--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
141+
}...)
142+
} else {
143+
c.iptablesRulesV4 = append(c.iptablesRulesV4, []iptablesRule{
144+
// Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
145+
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP,
146+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
147+
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP,
148+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
149+
// There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking,
150+
// need these additional filter table rules.
151+
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP,
152+
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
153+
{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP,
154+
"--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
155+
// Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
156+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
157+
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
158+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
159+
"--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
160+
// Additional filter table rules for traffic frpm localIp:localPort
161+
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
162+
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
163+
{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP,
164+
"--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}},
165+
// Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods
166+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
167+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
168+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP,
169+
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
170+
// skip connection tracking for healthcheck requests generated by liveness probe to health plugin
171+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP,
172+
"--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
173+
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
174+
"--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
175+
}...)
176+
}
151177

152-
func newIPTables(isIPv6 bool) utiliptables.Interface {
153-
protocol := utiliptables.ProtocolIPv4
154-
if isIPv6 {
155-
protocol = utiliptables.ProtocolIPv6
156178
}
157-
return utiliptables.New(protocol)
179+
c.iptablesV4 = utiliptables.New(utiliptables.ProtocolIPv4)
180+
c.iptablesV6 = utiliptables.New(utiliptables.ProtocolIPv6)
158181
}
159182

160183
func handleIPTablesError(err error) {
@@ -181,32 +204,66 @@ func (c *CacheApp) TeardownNetworking() error {
181204
err = c.netifHandle.RemoveDummyDevice(c.params.InterfaceName)
182205
}
183206
if c.params.SetupIptables {
184-
for _, rule := range c.iptablesRules {
207+
for _, rule := range c.iptablesRulesV4 {
208+
exists := true
209+
for exists == true {
210+
// check in a loop in case the same rule got added multiple times.
211+
err = c.iptablesV4.DeleteRule(rule.table, rule.chain, rule.args...)
212+
if err != nil {
213+
clog.Errorf("Failed deleting iptablesV4 rule %v, error - %v", rule, err)
214+
handleIPTablesError(err)
215+
}
216+
exists, err = c.iptablesV4.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
217+
if err != nil {
218+
clog.Errorf("Failed checking iptablesV4 rule after deletion, rule - %v, error - %v", rule, err)
219+
handleIPTablesError(err)
220+
}
221+
}
222+
// Delete the rule one last time since EnsureRule creates the rule if it doesn't exist
223+
err = c.iptablesV4.DeleteRule(rule.table, rule.chain, rule.args...)
224+
}
225+
for _, rule := range c.iptablesRulesV6 {
185226
exists := true
186227
for exists == true {
187228
// check in a loop in case the same rule got added multiple times.
188-
err = c.iptables.DeleteRule(rule.table, rule.chain, rule.args...)
229+
err = c.iptablesV6.DeleteRule(rule.table, rule.chain, rule.args...)
189230
if err != nil {
190-
clog.Errorf("Failed deleting iptables rule %v, error - %v", rule, err)
231+
clog.Errorf("Failed deleting iptablesV6 rule %v, error - %v", rule, err)
191232
handleIPTablesError(err)
192233
}
193-
exists, err = c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
234+
exists, err = c.iptablesV6.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
194235
if err != nil {
195-
clog.Errorf("Failed checking iptables rule after deletion, rule - %v, error - %v", rule, err)
236+
clog.Errorf("Failed checking iptablesV6 rule after deletion, rule - %v, error - %v", rule, err)
196237
handleIPTablesError(err)
197238
}
198239
}
199240
// Delete the rule one last time since EnsureRule creates the rule if it doesn't exist
200-
err = c.iptables.DeleteRule(rule.table, rule.chain, rule.args...)
241+
err = c.iptablesV6.DeleteRule(rule.table, rule.chain, rule.args...)
201242
}
202243
}
203244
return err
204245
}
205246

206247
func (c *CacheApp) setupNetworking() {
207248
if c.params.SetupIptables {
208-
for _, rule := range c.iptablesRules {
209-
exists, err := c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
249+
for _, rule := range c.iptablesRulesV4 {
250+
exists, err := c.iptablesV4.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
251+
switch {
252+
case exists:
253+
// debug messages can be printed by including "debug" plugin in coreFile.
254+
clog.Debugf("iptables rule %v for nodelocaldns already exists", rule)
255+
continue
256+
case err == nil:
257+
clog.Infof("Added back nodelocaldns rule - %v", rule)
258+
continue
259+
default:
260+
// iptables check/rule add failed with error since control reached here.
261+
clog.Errorf("Error checking/adding iptables rule %v, error - %v", rule, err)
262+
handleIPTablesError(err)
263+
}
264+
}
265+
for _, rule := range c.iptablesRulesV6 {
266+
exists, err := c.iptablesV6.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
210267
switch {
211268
case exists:
212269
// debug messages can be printed by including "debug" plugin in coreFile.

‎cmd/node-cache/main.go‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ import (
2525

2626
corednsmain "github.com/coredns/coredns/coremain"
2727
clog "github.com/coredns/coredns/plugin/pkg/log"
28-
utilnet "k8s.io/utils/net"
2928

3029
"github.com/coredns/caddy"
3130
// blank imports to make sure the plugin code is pulled in from vendor when building node-cache image
@@ -101,12 +100,6 @@ func parseAndValidateFlags() (*app.ConfigParams, error) {
101100
params.LocalIPs = append(params.LocalIPs, newIP)
102101
}
103102

104-
// validate all the IPs have the same IP family
105-
for _, ip := range params.LocalIPs {
106-
if utilnet.IsIPv6(params.LocalIPs[0]) != utilnet.IsIPv6(ip) {
107-
return params, fmt.Errorf("unexpected IP Family for localIP - %q, want IPv6=%v", ip, utilnet.IsIPv6(params.LocalIPs[0]))
108-
}
109-
}
110103
// lookup specified dns port
111104
f := flag.Lookup("dns.port")
112105
if f == nil {

0 commit comments

Comments
 (0)