Skip to content

Commit 4b8de9e

Browse files
YujiOshimaYujiOshima
committed
Experimental: Advertise container network with BGP in ipvlan l3
1 parent 3f771e3 commit 4b8de9e

File tree

9 files changed

+828
-6
lines changed

9 files changed

+828
-6
lines changed

‎drivers/ipvlan/ipvlan.go‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,16 @@ const (
1515
vethLen = 7
1616
containerVethPrefix = "eth"
1717
vethPrefix = "veth"
18-
ipvlanType = "ipvlan" // driver type name
19-
modeL2 = "l2" // ipvlan mode l2 is the default
20-
modeL3 = "l3" // ipvlan L3 mode
21-
parentOpt = "parent" // parent interface -o parent
22-
modeOpt = "_mode" // ipvlan mode ux opt suffix
18+
ipvlanType = "ipvlan" // driver type name
19+
modeL2 = "l2" // ipvlan mode l2 is the default
20+
modeL3 = "l3" // ipvlan L3 mode
21+
parentOpt = "parent" // parent interface -o parent
22+
modeOpt = "_mode" // ipvlan mode ux opt suffix
23+
bgpNeighborOpt = "bgp-neighbor" // BGP neighbor address
24+
vrfOpt = "vrf" // BGP vrf ID
25+
asOpt = "asnum" // BGP AS number default 65000
26+
remoteAsOpt = "rasnum" // BGP remote AS number dafault 65000
27+
subnetAdvertise = "subnet-advertise" // Advertise IP Subnet with BGP
2328
)
2429

2530
var driverModeOpt = ipvlanType + modeOpt // mode -o ipvlan_mode

‎drivers/ipvlan/ipvlan_joinleave.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,14 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
5959
if err := jinfo.AddStaticRoute(defaultRoute.Destination, defaultRoute.RouteType, defaultRoute.NextHop); err != nil {
6060
return fmt.Errorf("failed to set an ipvlan l3 mode ipv4 default gateway: %v", err)
6161
}
62+
if n.config.SubnetAdvertise == "" {
63+
//Advertise container route as /32 route
64+
advIP := &net.IPNet{IP: ep.addr.IP, Mask: net.IPv4Mask(255, 255, 255, 255)}
65+
err = routemanager.AdvertiseNewRoute(advIP.String(), n.config.VrfID)
66+
if err != nil {
67+
return err
68+
}
69+
}
6270
logrus.Debugf("Ipvlan Endpoint Joined with IPv4_Addr: %s, Ipvlan_Mode: %s, Parent: %s",
6371
ep.addr.IP.String(), n.config.IpvlanMode, n.config.Parent)
6472
// If the endpoint has a v6 address, set a v6 default route
@@ -130,6 +138,11 @@ func (d *driver) Leave(nid, eid string) error {
130138
if err != nil {
131139
return err
132140
}
141+
if network.config.IpvlanMode == modeL3 && network.config.SubnetAdvertise == "" {
142+
//Withdraw container route as /32 route
143+
witdIP := &net.IPNet{IP: endpoint.addr.IP, Mask: net.IPv4Mask(255, 255, 255, 255)}
144+
err = routemanager.WithdrawRoute(witdIP.String(), network.config.VrfID)
145+
}
133146
if endpoint == nil {
134147
return fmt.Errorf("could not find endpoint with id %s", eid)
135148
}

‎drivers/ipvlan/ipvlan_network.go‎

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package ipvlan
22

33
import (
44
"fmt"
5-
65
"github.com/Sirupsen/logrus"
76
"github.com/docker/docker/pkg/parsers/kernel"
87
"github.com/docker/docker/pkg/stringid"
@@ -11,6 +10,7 @@ import (
1110
"github.com/docker/libnetwork/options"
1211
"github.com/docker/libnetwork/osl"
1312
"github.com/docker/libnetwork/types"
13+
"strconv"
1414
)
1515

1616
// CreateNetwork the network for the specified driver type
@@ -113,6 +113,34 @@ func (d *driver) createNetwork(config *configuration) error {
113113
endpoints: endpointTable{},
114114
config: config,
115115
}
116+
if config.IpvlanMode == modeL3 {
117+
if routemanager == nil {
118+
InitRouteMonitering(config.ASnum, config.RemoteASnum)
119+
}
120+
err := routemanager.CreateVrfNetwork(config.Parent, config.VrfID)
121+
if err != nil {
122+
return err
123+
}
124+
if config.BgpNeighbor != "" {
125+
routemanager.DiscoverNew(false, config.BgpNeighbor)
126+
}
127+
if config.SubnetAdvertise != "" {
128+
if config.Ipv4Subnets != nil {
129+
for _, subnet := range config.Ipv4Subnets {
130+
err := routemanager.AdvertiseNewRoute(subnet.SubnetIP, config.VrfID)
131+
if err != nil {
132+
return err
133+
}
134+
}
135+
for _, subnet := range config.Ipv6Subnets {
136+
err := routemanager.AdvertiseNewRoute(subnet.SubnetIP, config.VrfID)
137+
if err != nil {
138+
return err
139+
}
140+
}
141+
}
142+
}
143+
}
116144
// add the *network
117145
d.addNetwork(n)
118146

@@ -147,6 +175,23 @@ func (d *driver) DeleteNetwork(nid string) error {
147175
}
148176
}
149177
}
178+
if n.config.SubnetAdvertise != "" {
179+
//Advertise container network subnet
180+
if n.config.Ipv4Subnets != nil {
181+
for _, subnet := range n.config.Ipv4Subnets {
182+
err := routemanager.WithdrawRoute(subnet.SubnetIP, n.config.VrfID)
183+
if err != nil {
184+
return err
185+
}
186+
}
187+
for _, subnet := range n.config.Ipv6Subnets {
188+
err := routemanager.WithdrawRoute(subnet.SubnetIP, n.config.VrfID)
189+
if err != nil {
190+
return err
191+
}
192+
}
193+
}
194+
}
150195
// delete the *network
151196
d.deleteNetwork(nid)
152197
// delete the network record from persistent cache
@@ -211,6 +256,26 @@ func (config *configuration) fromOptions(labels map[string]string) error {
211256
case driverModeOpt:
212257
// parse driver option '-o ipvlan_mode'
213258
config.IpvlanMode = value
259+
case bgpNeighborOpt:
260+
// parse driver option '-o bgp-neighbor'
261+
config.BgpNeighbor = value
262+
case vrfOpt:
263+
// parse driver option '-o vrf'
264+
_, err := strconv.Atoi(value)
265+
if err != nil {
266+
logrus.Errorf("vrf ID must be numeral")
267+
return err
268+
}
269+
config.VrfID = value
270+
case asOpt:
271+
// parse driver options '-o asnum'
272+
config.ASnum = value
273+
case remoteAsOpt:
274+
// parse driver options '-o rasnum'
275+
config.RemoteASnum = value
276+
case subnetAdvertise:
277+
// set driver options '-o subnet-advertise'
278+
config.SubnetAdvertise = "True"
214279
}
215280
}
216281
return nil

‎drivers/ipvlan/ipvlan_store.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ type configuration struct {
2525
CreatedSlaveLink bool
2626
Ipv4Subnets []*ipv4Subnet
2727
Ipv6Subnets []*ipv6Subnet
28+
BgpNeighbor string
29+
VrfID string
30+
ASnum string
31+
RemoteASnum string
32+
SubnetAdvertise string
2833
}
2934

3035
type ipv4Subnet struct {

‎drivers/ipvlan/l3_del_routes.go‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
package ipvlan
2+
3+
import (
4+
"fmt"
5+
log "github.com/Sirupsen/logrus"
6+
"github.com/vishvananda/netlink"
7+
"net"
8+
)
9+
10+
// Cleanup links with netlink syscalls with a scope of:
11+
// RT_SCOPE_LINK = 0xfd (253)
12+
// RT_SCOPE_UNIVERSE = 0x0 (0)
13+
func cleanExistingRoutes(ifaceStr string) error {
14+
iface, err := netlink.LinkByName(ifaceStr)
15+
ipvlanParentIface, err := netlink.LinkByName(ifaceStr)
16+
if err != nil {
17+
log.Errorf("Error occoured finding the linux link [ %s ] from netlink: %s", ipvlanParentIface.Attrs().Name, err)
18+
return err
19+
}
20+
routes, err := netlink.RouteList(iface, netlink.FAMILY_V4)
21+
if err != nil {
22+
log.Errorf("Unable to retreive netlink routes: %s", err)
23+
return err
24+
}
25+
ifaceIP, err := getIfaceIP(ifaceStr)
26+
if err != nil {
27+
log.Errorf("Unable to retreive a usable IP via ethernet interface: %s", ifaceStr)
28+
return err
29+
}
30+
for _, route := range routes {
31+
if route.Dst == nil {
32+
log.Debugf("Ignoring route [ %v ] Dst is nil", route)
33+
continue
34+
}
35+
if netOverlaps(ifaceIP, route.Dst) == true {
36+
log.Debugf("Ignoring route [ %v ] as it is associated to the [ %s ] interface", ifaceIP, ifaceStr)
37+
} else if route.Scope == 0x0 || route.Scope == 0xfd {
38+
// Remove link and universal routes from the docker host ipvlan interface
39+
log.Debugf("Cleaning static route cache for the destination: [ %s ]", route.Dst)
40+
err := delRoute(route, ipvlanParentIface)
41+
if err != nil {
42+
log.Errorf("Error deleting static route cache for Destination: [ %s ] and Nexthop [ %s ] Error: %s", route.Dst, route.Gw, err)
43+
}
44+
}
45+
}
46+
return nil
47+
}
48+
49+
func verifyRoute(bgpRoute *net.IPNet) {
50+
networks, err := netlink.RouteList(nil, netlink.FAMILY_V4)
51+
if err != nil {
52+
return
53+
}
54+
for _, network := range networks {
55+
if network.Dst != nil && netOverlaps(bgpRoute, network.Dst) {
56+
log.Errorf("The network [ %v ] learned via BGP conflicts with an existing route on this host [ %v ]", bgpRoute, network.Dst)
57+
return
58+
}
59+
}
60+
return
61+
}
62+
63+
func netOverlaps(netX *net.IPNet, netY *net.IPNet) bool {
64+
if firstIP, _ := networkRange(netX); netY.Contains(firstIP) {
65+
return true
66+
}
67+
if firstIP, _ := networkRange(netY); netX.Contains(firstIP) {
68+
return true
69+
}
70+
return false
71+
}
72+
73+
func networkRange(network *net.IPNet) (net.IP, net.IP) {
74+
var (
75+
netIP = network.IP.To4()
76+
firstIP = netIP.Mask(network.Mask)
77+
lastIP = net.IPv4(0, 0, 0, 0).To4()
78+
)
79+
80+
for i := 0; i < len(lastIP); i++ {
81+
lastIP[i] = netIP[i] | ^network.Mask[i]
82+
}
83+
return firstIP, lastIP
84+
}
85+
86+
func getIfaceIP(name string) (*net.IPNet, error) {
87+
iface, err := netlink.LinkByName(name)
88+
if err != nil {
89+
return nil, err
90+
}
91+
addrs, err := netlink.AddrList(iface, netlink.FAMILY_V4)
92+
if err != nil {
93+
return nil, err
94+
}
95+
if len(addrs) == 0 {
96+
return nil, fmt.Errorf("Interface %v has no IP addresses", name)
97+
}
98+
if len(addrs) > 1 {
99+
log.Debugf("Interface %v has more than 1 IPv4 address. Default is %v\n", name, addrs[0].IP)
100+
}
101+
return addrs[0].IPNet, nil
102+
}
103+
104+
// delRoute deletes any netlink route
105+
func delRoute(route netlink.Route, iface netlink.Link) error {
106+
return netlink.RouteDel(&netlink.Route{
107+
Scope: route.Scope,
108+
LinkIndex: iface.Attrs().Index,
109+
Dst: route.Dst,
110+
Gw: route.Gw,
111+
})
112+
}
113+
114+
// delRemoteRoute deletes a host-scoped route to a device.
115+
func delRemoteRoute(neighborNetwork *net.IPNet, nextHop net.IP, iface netlink.Link) error {
116+
return netlink.RouteDel(&netlink.Route{
117+
Scope: netlink.SCOPE_UNIVERSE,
118+
LinkIndex: iface.Attrs().Index,
119+
Dst: neighborNetwork,
120+
Gw: nextHop,
121+
})
122+
}

‎drivers/ipvlan/l3_route_common.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package ipvlan
2+
3+
import (
4+
"net"
5+
)
6+
7+
type ribCache struct {
8+
BgpTable map[string]*ribLocal
9+
}
10+
11+
// Unmarshalled BGP update binding for simplicity
12+
type ribLocal struct {
13+
BgpPrefix *net.IPNet
14+
OriginatorIP net.IP
15+
NextHop net.IP
16+
Age int
17+
Best bool
18+
IsWithdraw bool
19+
IsHostRoute bool
20+
IsLocal bool
21+
AsPath string
22+
RT string
23+
}
24+
25+
type ribTest struct {
26+
BgpTable map[string]*ribLocal
27+
}

‎drivers/ipvlan/l3_route_interface.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package ipvlan
2+
3+
var routemanager routingInterface
4+
5+
type host struct {
6+
isself bool
7+
Address string
8+
}
9+
10+
type routingInterface interface {
11+
CreateVrfNetwork(ParentIface string, vrfID string) error
12+
AdvertiseNewRoute(localPrefix string, vrfID string) error
13+
WithdrawRoute(localPrefix string, vrfID string) error
14+
DiscoverNew(isself bool, Address string) error
15+
DiscoverDelete(isself bool, Address string) error
16+
}
17+
18+
// InitRouteMonitering initialize and start maniternig routing table of host
19+
func InitRouteMonitering(as string, ras string) {
20+
routemanager = NewBgpRouteManager(as, ras)
21+
}

0 commit comments

Comments
 (0)