Verify source addresses of egress NAT traffic

docs/cmd-coil-egress.md

@@ -123,3 +123,27 @@ This value is from the result of `iptables -t filter -L -vn` or nftables equival
 | `pod`       | The pod name                  |
 | `protocol`  | The protocol name                 |
 
+### `coil_egress_nftables_peer_src_dropped_packets_total`
+
+This is the total number of packets from clients dropped by iptables/nftables in a egress NAT pod because their source addresses are not the IP addresses of the clients.
+This value is the sum of the counters of the rules commented as `coil-peer-src-*` in the result of `iptables -t filter -L FORWARD -vn` and `iptables -t filter -L INPUT -vn` or nftables equivalent.
+
+| Label       | Description                   |
+| ----------- | ----------------------------- |
+| `namespace` | The egress resource namespace |
+| `egress`    | The egress resource name      |
+| `pod`       | The pod name                  |
+| `protocol`  | The protocol name                 |
+
+### `coil_egress_nftables_peer_src_dropped_bytes_total`
+
+This is the total bytes of packets from clients dropped by iptables/nftables in a egress NAT pod because their source addresses are not the IP addresses of the clients.
+This value is the sum of the counters of the rules commented as `coil-peer-src-*` in the result of `iptables -t filter -L FORWARD -vn` and `iptables -t filter -L INPUT -vn` or nftables equivalent.
+
+| Label       | Description                   |
+| ----------- | ----------------------------- |
+| `namespace` | The egress resource namespace |
+| `egress`    | The egress resource name      |
+| `pod`       | The pod name                  |
+| `protocol`  | The protocol name                 |
+

docs/usage.md

@@ -22,6 +22,7 @@ For installation, read [setup.md](setup.md).
     - [Use NetworkPolicy to prohibit NAT usage](#use-networkpolicy-to-prohibit-nat-usage)
     - [Session affinity](#session-affinity)
     - [Use egress only for connections originating on the client](#use-egress-only-for-connections-originating-on-the-client)
+    - [Source address verification](#source-address-verification)
   - [Metrics](#metrics)
     - [How to scrape metrics](#how-to-scrape-metrics)
 
@@ -321,6 +322,24 @@ In case of incomming connections, the same interface will be used for egress tra
 e.g. if connection will be estabilished on `eth0`, the traffic will not be routed through `fou`,
 but will be handled by `eth0`.
 
+### Source address verification
+
+NAT servers forward packets from a client only when their source address is
+the IP address of the client Pod.  Other packets from the client are dropped.
+This prevents a client from spoofing the source address to make the NAT server
+deliver reply packets to another Pod.
+The number of dropped packets is exported as
+`coil_egress_nftables_peer_src_dropped_packets_total` metrics.
+
+If a client Pod forwards traffic of nested network namespaces or containers
+through Egress, the traffic must be SNATed to the Pod IP address before it
+leaves the Pod.  Docker's default masquerade rule does this, so containers
+run by Docker in a client Pod work without extra configuration.
+
+NAT servers also drop FoU packets whose outer source address is not the IP
+address of a client Pod.  Therefore, traffic from client Pods to the ClusterIP
+of the Egress Service must not be SNATed.
+
 ## Metrics
 
 Coil exposes two types of Prometheus metrics.

v2/cmd/coil-egress/sub/run.go

@@ -45,6 +45,8 @@ func init() {
 	metrics.Registry.MustRegister(egressMetrics.NfTableMasqueradePackets)
 	metrics.Registry.MustRegister(egressMetrics.NfTableInvalidBytes)
 	metrics.Registry.MustRegister(egressMetrics.NfTableInvalidPackets)
+	metrics.Registry.MustRegister(egressMetrics.NfTablePeerSrcDroppedBytes)
+	metrics.Registry.MustRegister(egressMetrics.NfTablePeerSrcDroppedPackets)
 }
 
 func subMain() error {
@@ -102,7 +104,7 @@ func subMain() error {
 	}
 
 	setupLog.Info("initialize FoU tunnel", "port", config.port, "ipv4", ipv4.String(), "ipv6", ipv6.String())
-	ft := fou.NewFoUTunnel(config.port, ipv4, ipv6, nil)
+	ft := fou.NewServerFoUTunnel(config.port, ipv4, ipv6, nil)
 	if err := ft.Init(); err != nil {
 		return err
 	}

v2/controllers/egress_watcher.go

@@ -198,7 +198,7 @@ func (r *EgressWatcher) hook(gwn gwNets, log *logr.Logger) func(ipv4, ipv6 net.I
 	return func(ipv4, ipv6 net.IP) error {
 		// We assume that coild already has configured NAT for the client,
 		// so we ensure that both FoUTunnel and NATClient have been initialized.
-		ft := fou.NewFoUTunnel(r.EgressPort, ipv4, ipv6, func(message string) {
+		ft := fou.NewClientFoUTunnel(r.EgressPort, ipv4, ipv6, func(message string) {
 			log.Info(message)
 		})
 		if !ft.IsInitialized() {
@@ -212,6 +212,10 @@ func (r *EgressWatcher) hook(gwn gwNets, log *logr.Logger) func(ipv4, ipv6 net.I
 			return fmt.Errorf("natClient hasn't been initialized: %w", err)
 		}
 
+		if err := cl.EnsureFlowBasedDevRules(); err != nil {
+			return err
+		}
+
 		link, err := ft.AddPeer(gwn.gateway, gwn.sportAuto)
 		if errors.Is(err, fou.ErrIPFamilyMismatch) {
 			// ignore unsupported IP family link

v2/controllers/pod_watcher.go

@@ -266,9 +266,14 @@ OUTER2:
 			}
 		}
 		logger.Info("delete peer", "caller", "addPod", "ip", eip.String(), "podIPs", podIPs, "existing", existing)
+		// Delete the link before the source address verification rule
+		// so that the link never forwards unverified packets.
 		if err := r.ft.DelPeer(eip); err != nil {
 			return err
 		}
+		if err := r.nat.DelClient(eip); err != nil {
+			return err
+		}
 		if n := ClientPodInfo.DeletePartialMatch(prometheus.Labels{"namespace": pod.GetNamespace(), "pod": pod.GetName(), "pod_ip": eip.String(), "egress": r.myName, "egress_namespace": r.myNS}); n != 1 {
 			logger.Error(errors.New("metrics deletion error"), "the number of deleted metrics is not one for", "pod", pod.GetName(), "namespace", pod.GetNamespace())
 		}
@@ -314,9 +319,13 @@ func (r *podWatcher) delPeer(n types.NamespacedName, caller, podPhase string, lo
 		}
 		if !existsLivePeers {
 			logger.Info("delete peer", "caller", caller, "ip", ip.String(), "podIPs", r.podAddrs[key], "podPhase", podPhase)
+			// See addPod for the order of deletion.
 			if err := r.ft.DelPeer(ip); err != nil {
 				return err
 			}
+			if err := r.nat.DelClient(ip); err != nil {
+				return err
+			}
 		}
 
 		if keySet, ok := r.peers[ip.String()]; ok {

v2/controllers/pod_watcher_test.go

@@ -238,7 +238,6 @@ var _ = Describe("Pod watcher", Ordered, func() {
 				"fd01::1":  {},
 				"10.1.1.2": {},
 				"fd01::2":  {},
-				"fd01::3":  {}, // nat.Server does not have remove API
 				"10.1.1.7": {},
 				"fd01::7":  {},
 			})
@@ -275,9 +274,7 @@ var _ = Describe("Pod watcher", Ordered, func() {
 
 		Eventually(func() bool {
 			return reflect.DeepEqual(nat.GetClients(), map[string]struct{}{
-				"10.1.1.2": {}, // cannot be removed
-				"fd01::2":  {}, // ditto
-				"fd01::3":  {},
+				"fd01::3": {},
 			})
 		}).Should(BeTrue())
 
@@ -333,6 +330,10 @@ var _ = Describe("Pod watcher", Ordered, func() {
 				"10.1.1.2": true,
 				"fd01::2":  true,
 				"fd01::3":  true,
+			}) && reflect.DeepEqual(nat.GetClients(), map[string]struct{}{
+				"10.1.1.2": {},
+				"fd01::2":  {},
+				"fd01::3":  {},
 			})
 		}).Should(BeTrue())
 
@@ -354,6 +355,11 @@ var _ = Describe("Pod watcher", Ordered, func() {
 				"fd01::2":  true,
 				"fd01::3":  true,
 				"10.1.1.5": true,
+			}) && reflect.DeepEqual(nat.GetClients(), map[string]struct{}{
+				"10.1.1.2": {},
+				"fd01::2":  {},
+				"fd01::3":  {},
+				"10.1.1.5": {},
 			})
 		}, 5*time.Second, 1*time.Second).Should(BeTrue())
 	})

v2/pkg/fou/fou.go

@@ -19,6 +19,13 @@ const (
 	FoU6LinkPrefix = "fou6_"
 )
 
+// Names of the flow based IPIP tunnel devices created for NAT clients.
+// See setupFlowBasedIP4TunDevice.
+const (
+	FlowBasedIP4TunDevice = "coil_ipip4"
+	FlowBasedIP6TunDevice = "coil_ipip6"
+)
+
 const fouDummy = "fou-dummy"
 
 func fouName(addr net.IP) string {
@@ -56,18 +63,49 @@ type FoUTunnel interface {
 	DelPeer(net.IP) error
 }
 
-// NewFoUTunnel creates a new FoUTunnel.
+// tunnelRole represents which side of the egress NAT a FoUTunnel works for.
+type tunnelRole int
+
+const (
+	// roleClient is for NAT clients, i.e. Pods using Egress.
+	roleClient tunnelRole = iota
+	// roleServer is for NAT servers, i.e. egress Pods.
+	roleServer
+)
+
+// NewClientFoUTunnel creates a new FoUTunnel for NAT clients.
+//
+// The tunnel sets up flow based IPIP tunnel devices to receive returning
+// packets from egress Pods, and does not enable IP forwarding.
+// See setupFlowBasedIP4TunDevice for details.
+//
 // port is the UDP port to receive FoU packets.
 // localIPv4 is the local IPv4 address of the IPIP tunnel.  This can be nil.
 // localIPv6 is the same as localIPv4 for IPv6.
-func NewFoUTunnel(port int, localIPv4, localIPv6 net.IP, logFunc func(string)) FoUTunnel {
+func NewClientFoUTunnel(port int, localIPv4, localIPv6 net.IP, logFunc func(string)) FoUTunnel {
+	return newFoUTunnel(roleClient, port, localIPv4, localIPv6, logFunc)
+}
+
+// NewServerFoUTunnel creates a new FoUTunnel for NAT servers.
+//
+// The tunnel enables IP forwarding to route packets from clients, and does not
+// set up flow based IPIP tunnel devices so that only packets from registered
+// peers are decapsulated.
+//
+// The parameters are the same as NewClientFoUTunnel.
+func NewServerFoUTunnel(port int, localIPv4, localIPv6 net.IP, logFunc func(string)) FoUTunnel {
+	return newFoUTunnel(roleServer, port, localIPv4, localIPv6, logFunc)
+}
+
+func newFoUTunnel(role tunnelRole, port int, localIPv4, localIPv6 net.IP, logFunc func(string)) FoUTunnel {
 	if localIPv4 != nil && localIPv4.To4() == nil {
 		panic("invalid IPv4 address")
 	}
 	if localIPv6 != nil && localIPv6.To4() != nil {
 		panic("invalid IPv6 address")
 	}
 	return &fouTunnel{
+		role:    role,
 		port:    port,
 		local4:  localIPv4,
 		local6:  localIPv6,
@@ -76,6 +114,7 @@ func NewFoUTunnel(port int, localIPv4, localIPv6 net.IP, logFunc func(string)) F
 }
 
 type fouTunnel struct {
+	role    tunnelRole
 	port    int
 	local4  net.IP
 	local6  net.IP
@@ -113,8 +152,11 @@ func (t *fouTunnel) Init() error {
 		if _, err := sysctl.Sysctl("net.ipv4.conf.all.rp_filter", "0"); err != nil {
 			return fmt.Errorf("setting net.ipv4.conf.all.rp_filter=0 failed: %w", err)
 		}
-		if err := ip.EnableIP4Forward(); err != nil {
-			return fmt.Errorf("failed to enable IPv4 forwarding: %w", err)
+		// NAT clients only send and receive their own packets.
+		if t.role == roleServer {
+			if err := ip.EnableIP4Forward(); err != nil {
+				return fmt.Errorf("failed to enable IPv4 forwarding: %w", err)
+			}
 		}
 	}
 	if t.local6 != nil {
@@ -130,8 +172,10 @@ func (t *fouTunnel) Init() error {
 		if err != nil {
 			return fmt.Errorf("netlink: fou add failed: %w", err)
 		}
-		if err := ip.EnableIP6Forward(); err != nil {
-			return fmt.Errorf("failed to enable IPv6 forwarding: %w", err)
+		if t.role == roleServer {
+			if err := ip.EnableIP6Forward(); err != nil {
+				return fmt.Errorf("failed to enable IPv6 forwarding: %w", err)
+			}
 		}
 
 		// avoid any existing DROP rule by rpfilter extension.
@@ -175,8 +219,10 @@ func (t *fouTunnel) addPeer4(addr net.IP, sportAuto bool) (netlink.Link, error)
 		return nil, err
 	}
 
-	if err := setupFlowBasedIP4TunDevice(); err != nil {
-		return nil, fmt.Errorf("netlink: failed to setup ipip device: %w", err)
+	if t.role == roleClient {
+		if err := setupFlowBasedIP4TunDevice(); err != nil {
+			return nil, fmt.Errorf("netlink: failed to setup ipip device: %w", err)
+		}
 	}
 
 	return netlink.LinkByName(linkName)
@@ -249,8 +295,10 @@ func (t *fouTunnel) addPeer6(addr net.IP, sportAuto bool) (netlink.Link, error)
 		return nil, err
 	}
 
-	if err := setupFlowBasedIP6TunDevice(); err != nil {
-		return nil, fmt.Errorf("netlink: failed to setup ipip device: %w", err)
+	if t.role == roleClient {
+		if err := setupFlowBasedIP6TunDevice(); err != nil {
+			return nil, fmt.Errorf("netlink: failed to setup ipip device: %w", err)
+		}
 	}
 
 	return netlink.LinkByName(linkName)
@@ -312,6 +360,14 @@ func (t *fouTunnel) addOrRecreatePeer6(addr net.IP, sportAuto bool) (string, err
 // This flow based IPIP tunnel device is used to decapsulate packets from
 // the router Pods.
 //
+// It is needed only for NAT clients.  A client sends packets to the Service's
+// ClusterIP, but the returning packets come from the router Pod's IP address,
+// so they do not match the IPIP tunnel device configured for the ClusterIP.
+// Since this device accepts packets from any source, NAT servers must not
+// create it; packets from clients always match the per-peer tunnel devices.
+// On NAT servers, the fallback devices are left as they are.  They are down
+// and never used for decapsulation.
+//
 // Calling this function may result in tunl0 (v4) or ip6tnl0 (v6)
 // fallback interface being renamed to coil_tunl or coil_ip6tnl.
 // This is to communicate to the user that this plugin has taken
@@ -326,7 +382,7 @@ func (t *fouTunnel) addOrRecreatePeer6(addr net.IP, sportAuto bool) (string, err
 // By default, these interfaces will be created in new network namespaces,
 // but this behavior can be disabled by setting net.core.fb_tunnels_only_for_init_net = 2.
 func setupFlowBasedIP4TunDevice() error {
-	ipip4Device := "coil_ipip4"
+	ipip4Device := FlowBasedIP4TunDevice
 	// Set up IPv4 tunnel device if requested.
 	if err := setupDevice(&netlink.Iptun{
 		LinkAttrs: netlink.LinkAttrs{Name: ipip4Device},
@@ -346,7 +402,7 @@ func setupFlowBasedIP4TunDevice() error {
 
 // See setupFlowBasedIP4TunDevice
 func setupFlowBasedIP6TunDevice() error {
-	ipip6Device := "coil_ipip6"
+	ipip6Device := FlowBasedIP6TunDevice
 
 	// Set up IPv6 tunnel device if requested.
 	if err := setupDevice(&netlink.Ip6tnl{