[Swan] No ipsec0 device with XFRMi

Antony Antony antony at phenome.org
Mon Aug 10 20:48:23 UTC 2020


Hi Wolfgang,
Thanks for the testcase. Unfortunately, north has no second uplink/interface 
to reach east. So the test can't send the traffic yet. Now we can verify  
rules and verify "ip x s" mark/mask. Let me see if there is another way to 
test to able to send traffic with fwmark.  Add another rule or something,  
change http  to "nc" as a listener on east.

Tuomo, do you have any ideas to fix this test case? simulate two uplink or 
fwmark?

I would the patch more generic, where you can configure output mark. Then 
the mark is independent of if_id, for advanced routing usecase this would be 
better. Could you test the attached patch?  I am not sure I got mark 
correct, 8 LSB?

in the config 
mark-out=3/0xffffff

ip x s # should show the outputmark with mask[1]
src 192.1.2.23 dst 192.1.3.33
       proto esp spi 0xSPISPI reqid REQID mode tunnel
       replay-window 32 flag af-unspec
       output-mark 0x3/0xffffff
       aead rfc4106(gcm(aes)) 0xENCAUTHKEY 128
       if_id 0x1

[1] iproute2 mask
along the way I noticed iproute2 does not print the output mark's mask.
I have a quick patch for iproute 2 to print it.
https://github.com/antonyantony/iproute2/commit/69d7df795372a4f6737f9133b80120e2727f1ff3

And the updated testcase: 
https://github.com/antonyantony/libreswan/tree/xfrmi-outmark/testing/pluto/ikev2-xfrmi-14-fwmark

On Mon, Aug 10, 2020 at 10:41:35AM +0200, Wolfgang Nothdurft wrote:
> Am 30.07.20 um 07:57 schrieb Antony Antony:
> 
> > Can you can help create a testcase with fwmark and xfrmi?  you are using
> > marks with KLIPS? so it is not really configured in ipsec.conf? I wonder how
> > that would translate one-to-one.
> > 
> 
> Attached you can find an simplified testcase that corresponds approximately
> to what we do.
> 
> In this case marking http traffic, to route it on an other interface.
> 
> iptables -t mangle -I OUTPUT -p tcp --dport 80 -j MARK --set-mark 0x1
> ip ru add prio 1 fwmark 0x1 table 1
> ip r add default dev eth0 table 1
> 
> This case passes with my example patch when mapping the fwmark to 0x1000000.
> 
> Wolfgang

> commit c5468a72eea2316bf246ba521f17e5f833db9395
> Author: Wolfgang <build at localhost.localdomain>
> Date:   Mon Aug 10 04:29:15 2020 -0400
> 
>     * prototype testcase for conflicting fwmark with xfrmi
> 
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/description.txt b/testing/pluto/ikev2-xfrmi-14-fwmark/description.txt
> new file mode 100644
> index 0000000000..d68af1b1ca
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/description.txt
> @@ -0,0 +1 @@
> +The default XFRMi FWMARK conflicts with a policy based route.
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/east.console.txt b/testing/pluto/ikev2-xfrmi-14-fwmark/east.console.txt
> new file mode 100644
> index 0000000000..89ec07aef6
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/east.console.txt
> @@ -0,0 +1,20 @@
> +/testing/guestbin/swan-prep
> +east #
> + ipsec start
> +Redirecting to: [initsystem]
> +east #
> + /testing/pluto/bin/wait-until-pluto-started
> +east #
> + ipsec auto --add northnet-eastnet
> +002 added IKEv2 connection "northnet-eastnet"
> +east #
> + echo "initdone"
> +initdone
> +east #
> + ipsec whack --trafficstatus
> +006 #2: "northnet-eastnet", type=ESP, add_time=1234567890, inBytes=336, outBytes=336, id='@north'
> +east #
> + ../bin/check-for-core.sh
> +east #
> + if [ -f /sbin/ausearch ]; then ausearch -r -m avc -ts recent ; fi
> +
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/east.secrets b/testing/pluto/ikev2-xfrmi-14-fwmark/east.secrets
> new file mode 100644
> index 0000000000..7b53b85edb
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/east.secrets
> @@ -0,0 +1 @@
> + at east @north : PSK "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/eastinit.sh b/testing/pluto/ikev2-xfrmi-14-fwmark/eastinit.sh
> new file mode 100755
> index 0000000000..6dc310df8f
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/eastinit.sh
> @@ -0,0 +1,5 @@
> +/testing/guestbin/swan-prep
> +ipsec start
> +/testing/pluto/bin/wait-until-pluto-started
> +ipsec auto --add northnet-eastnet
> +echo "initdone"
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/final.sh b/testing/pluto/ikev2-xfrmi-14-fwmark/final.sh
> new file mode 100755
> index 0000000000..35dd99a15b
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/final.sh
> @@ -0,0 +1,7 @@
> +ipsec whack --trafficstatus
> +: ==== cut ====
> +ipsec auto --status
> +: ==== tuc ====
> +../bin/check-for-core.sh
> +if [ -f /sbin/ausearch ]; then ausearch -r -m avc -ts recent ; fi
> +: ==== end ====
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/ipsec.conf b/testing/pluto/ikev2-xfrmi-14-fwmark/ipsec.conf
> new file mode 100644
> index 0000000000..67cd088c14
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/ipsec.conf
> @@ -0,0 +1,31 @@
> +# /etc/ipsec.conf - Libreswan IPsec configuration file
> +
> +version 2.0
> +
> +config setup
> +	logfile=/tmp/pluto.log
> +	logtime=no
> +	logappend=no
> +	plutodebug="all"
> +	protostack=netkey
> +	dumpdir=/tmp
> +
> +conn %default
> +	authby=secret
> +	ikev2=insist
> +
> +conn base
> +	rightid=@east
> +	leftid=@north
> +	left=192.1.3.33
> +	right=192.1.2.23
> +	leftsubnet=192.0.3.0/24
> +
> +conn northnet-eastnet
> +	also=base
> +	ipsec-interface=no
> +
> +conn north
> +	also=base
> +	priority=3
> +	ipsec-interface=yes
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/nic.console.txt b/testing/pluto/ikev2-xfrmi-14-fwmark/nic.console.txt
> new file mode 100644
> index 0000000000..fcd5477274
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/nic.console.txt
> @@ -0,0 +1,7 @@
> +iptables -t nat -F
> +nic #
> + iptables -F
> +nic #
> + echo "initdone"
> +initdone
> +
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/nicinit.sh b/testing/pluto/ikev2-xfrmi-14-fwmark/nicinit.sh
> new file mode 100755
> index 0000000000..d48b3f9ad5
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/nicinit.sh
> @@ -0,0 +1,4 @@
> +iptables -t nat -F
> +iptables -F
> +echo "initdone"
> +: ==== end ====
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/north.console.txt b/testing/pluto/ikev2-xfrmi-14-fwmark/north.console.txt
> new file mode 100644
> index 0000000000..ecade8b2c4
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/north.console.txt
> @@ -0,0 +1,93 @@
> +/testing/guestbin/swan-prep
> +north #
> + iptables -t mangle -I OUTPUT -p tcp --dport 80 -j MARK --set-mark 0x1
> +north #
> + ip ru add prio 1 fwmark 0x1 table 1
> +north #
> + ip r add default dev eth0 table 1
> +north #
> + # this route from /etc/sysconfig/network-scripts/route-eth1 interfears
> +north #
> + ip route get to 192.0.2.254 | grep eth1 && ip route del 192.0.2.0/24 via 192.1.3.254 dev eth1
> +192.0.2.254 via 192.1.3.254 dev eth1 src 192.1.3.33 uid 0
> +RTNETLINK answers: No such process
> +north #
> + # ip link show ipsec1 2>/dev/null && ip link del ipsec1
> +north #
> + echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter
> +north #
> + ipsec start
> +Redirecting to: [initsystem]
> +north #
> + /testing/pluto/bin/wait-until-pluto-started
> +north #
> + ipsec auto --add north
> +002 added IKEv2 connection "north"
> +north #
> + echo "initdone"
> +initdone
> +north #
> + ipsec auto --up north
> +1v2 "north" #1: initiating IKEv2 connection
> +1v2 "north" #1: sent IKE_SA_INIT request
> +1v2 "north" #1: sent IKE_AUTH request {auth=IKEv2 cipher=AES_GCM_16_256 integ=n/a prf=HMAC_SHA2_512 group=MODP2048}
> +002 "north" #2: IKEv2 mode peer ID is ID_FQDN: '@east'
> +003 "north" #1: authenticated using authby=secret
> +002 "north" #2: negotiated connection [192.0.3.0-192.0.3.255:0-65535 0] -> [192.1.2.23-192.1.2.23:0-65535 0]
> +004 "north" #2: IPsec SA established tunnel mode {ESP=>0xESPESP <0xESPESP xfrm=AES_GCM_16_256-NONE NATOA=none NATD=none DPD=passive}
> +north #
> + # comments bellow are to understand/explore the basics : what is going on
> +north #
> + # ip link add ipsec1 type xfrm xfrmi-id 1 dev eth0
> +north #
> + # ip link set ipsec1 up
> +north #
> + # ip route add 192.0.2.0/24 dev ipsec1 src 192.0.3.254
> +north #
> + # tcpdump -s 0 -n -w /tmp/ipsec1.pcap -i ipsec1 & echo $! > /tmp/tcpdump.pid
> +north #
> + sleep  2
> +north #
> + ping -w 4 -c 4 192.1.2.23
> +PING 192.1.2.23 (192.1.2.23) 56(84) bytes of data.
> +64 bytes from 192.1.2.23: icmp_seq=1 ttl=64 time=0.XXX ms
> +64 bytes from 192.1.2.23: icmp_seq=2 ttl=64 time=0.XXX ms
> +64 bytes from 192.1.2.23: icmp_seq=3 ttl=64 time=0.XXX ms
> +64 bytes from 192.1.2.23: icmp_seq=4 ttl=64 time=0.XXX ms
> +--- 192.1.2.23 ping statistics ---
> +4 packets transmitted, 4 received, 0% packet loss, time XXXX
> +rtt min/avg/max/mdev = 0.XXX/0.XXX/0.XXX/0.XXX ms
> +north #
> + ip -s link show ipsec1
> +X: ipsec1 at eth1: <NOARP,UP,LOWER_UP> mtu 1500 state UNKNOWN
> +    RX: bytes  packets  errors  dropped overrun mcast   
> +    336        4        0       0       0       0       
> +    TX: bytes  packets  errors  dropped carrier collsns 
> +    336        4        0       0       0       0       
> +north #
> + #kill -9 $(cat /tmp/tcpdump.pid)
> +north #
> + sleep 2
> +north #
> + #cp /tmp/ipsec1.pcap OUTPUT/
> +north #
> + ip rule show
> +0:	from all lookup local
> +1:	from all fwmark 0x1 lookup 1
> +100:	from all to 192.1.2.23 fwmark 0x1000000 lookup 50
> +32766:	from all lookup main
> +32767:	from all lookup default
> +north #
> + ip route show table 50
> +192.1.2.23 dev eth1 scope link
> +north #
> + echo done
> +done
> +north #
> + ipsec whack --trafficstatus
> +006 #2: "north", type=ESP, add_time=1234567890, inBytes=336, outBytes=336, id='@east'
> +north #
> + ../bin/check-for-core.sh
> +north #
> + if [ -f /sbin/ausearch ]; then ausearch -r -m avc -ts recent ; fi
> +
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/north.secrets b/testing/pluto/ikev2-xfrmi-14-fwmark/north.secrets
> new file mode 100644
> index 0000000000..7b53b85edb
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/north.secrets
> @@ -0,0 +1 @@
> + at east @north : PSK "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/northinit.sh b/testing/pluto/ikev2-xfrmi-14-fwmark/northinit.sh
> new file mode 100755
> index 0000000000..96c6dd3592
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/northinit.sh
> @@ -0,0 +1,12 @@
> +/testing/guestbin/swan-prep
> +iptables -t mangle -I OUTPUT -p tcp --dport 80 -j MARK --set-mark 0x1
> +ip ru add prio 1 fwmark 0x1 table 1
> +ip r add default dev eth0 table 1
> +# this route from /etc/sysconfig/network-scripts/route-eth1 interfears
> +ip route get to 192.0.2.254 | grep eth1 && ip route del 192.0.2.0/24 via 192.1.3.254 dev eth1
> +# ip link show ipsec1 2>/dev/null && ip link del ipsec1
> +echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter
> +ipsec start
> +/testing/pluto/bin/wait-until-pluto-started
> +ipsec auto --add north
> +echo "initdone"
> diff --git a/testing/pluto/ikev2-xfrmi-14-fwmark/northrun.sh b/testing/pluto/ikev2-xfrmi-14-fwmark/northrun.sh
> new file mode 100755
> index 0000000000..caa3888ebb
> --- /dev/null
> +++ b/testing/pluto/ikev2-xfrmi-14-fwmark/northrun.sh
> @@ -0,0 +1,15 @@
> +ipsec auto --up north
> +# comments bellow are to understand/explore the basics : what is going on
> +# ip link add ipsec1 type xfrm xfrmi-id 1 dev eth0
> +# ip link set ipsec1 up
> +# ip route add 192.0.2.0/24 dev ipsec1 src 192.0.3.254
> +# tcpdump -s 0 -n -w /tmp/ipsec1.pcap -i ipsec1 & echo $! > /tmp/tcpdump.pid
> +sleep  2
> +ping -w 4 -c 4 192.1.2.23
> +ip -s link show ipsec1
> +#kill -9 $(cat /tmp/tcpdump.pid)
> +sleep 2
> +#cp /tmp/ipsec1.pcap OUTPUT/
> +ip rule show
> +ip route show table 50
> +echo done



-------------- next part --------------
>From db40495a9713bdef96d5db0feaa75bee59502e2a Mon Sep 17 00:00:00 2001
From: Antony Antony <antony at phenome.org>
Date: Mon, 10 Aug 2020 19:50:07 +0000
Subject: [PATCH 1/2] pluto: xfrmi add configuratio mark

mark-out=<mark>/<mask>
---
 programs/pluto/kernel.c      | 17 +++++++----
 programs/pluto/kernel.h      |  1 +
 programs/pluto/kernel_xfrm.c | 56 +++++++++++++++++++++++++++---------
 3 files changed, 56 insertions(+), 18 deletions(-)

diff --git a/programs/pluto/kernel.c b/programs/pluto/kernel.c
index 07bb2740d6..560501c708 100644
--- a/programs/pluto/kernel.c
+++ b/programs/pluto/kernel.c
@@ -565,14 +565,18 @@ static void jam_common_shell_out(struct jambuf *buf, const struct connection *c,
 		jam(buf, "CONNMARK_IN=%" PRIu32 "/%#08" PRIx32 " ",
 		    c->sa_marks.in.val, c->sa_marks.in.mask);
 	}
-	if (c->sa_marks.out.val != 0) {
+	if (c->sa_marks.out.val != 0 && c->xfrmi == NULL) {
 		jam(buf, "CONNMARK_OUT=%" PRIu32 "/%#08" PRIx32 " ",
 		    c->sa_marks.out.val, c->sa_marks.out.mask);
 	}
-	if (c->xfrmi != NULL && c->xfrmi->if_id > 0) {
-		if (addrinsubnet(&sr->that.host_addr, &sr->that.client)) {
+	if (c->xfrmi != NULL) {
+		if (c->sa_marks.out.val != 0) {
+			/* user configured XFRMI_SET_MARK (a.k.a. output mark) add it */
+			jam(buf, "PLUTO_XFRMI_FWMARK='%" PRIu32 "/%#08" PRIx32 "' ",
+			    c->sa_marks.out.val, c->sa_marks.out.mask);
+		} else if (addrinsubnet(&sr->that.host_addr, &sr->that.client)) {
 			jam(buf, "PLUTO_XFRMI_FWMARK='%" PRIu32 "/0xffffffff' ",
-					c->xfrmi->if_id);
+			    c->xfrmi->if_id);
 		} else {
 			address_buf bpeer;
 			subnet_buf peerclient_str;
@@ -1940,8 +1944,11 @@ static bool setup_half_ipsec_sa(struct state *st, bool inbound)
 		said_next->replay_window = c->sa_replay_window;
 		dbg("setting IPsec SA replay-window to %d", c->sa_replay_window);
 
-		if (c->xfrmi != NULL)
+		if (c->xfrmi != NULL) {
 			said_next->xfrm_if_id = c->xfrmi->if_id;
+			if (c->sa_marks.out.val != 0 || c->sa_marks.out.mask != 0)
+				said_next->mark_set = c->sa_marks.out;
+		}
 
 		if (!inbound && c->sa_tfcpad != 0 && !st->st_seen_no_tfc) {
 			dbg("Enabling TFC at %d bytes (up to PMTU)", c->sa_tfcpad);
diff --git a/programs/pluto/kernel.h b/programs/pluto/kernel.h
index 659e306f73..e63de76dde 100644
--- a/programs/pluto/kernel.h
+++ b/programs/pluto/kernel.h
@@ -172,6 +172,7 @@ struct kernel_sa {
 	struct xfrm_user_sec_ctx_ike *sec_ctx;
 	const char *nic_offload_dev;
 	uint32_t xfrm_if_id;
+	struct sa_mark mark_set; /* config keyword mark-out */
 
 	deltatime_t sa_lifetime; /* number of seconds until SA expires */
 };
diff --git a/programs/pluto/kernel_xfrm.c b/programs/pluto/kernel_xfrm.c
index cecfe8c535..641b89cd56 100644
--- a/programs/pluto/kernel_xfrm.c
+++ b/programs/pluto/kernel_xfrm.c
@@ -723,7 +723,7 @@ static bool netlink_raw_eroute(const ip_address *this_host,
 		{
 			struct sa_mark sa_mark = (dir == XFRM_POLICY_IN) ? sa_marks->in : sa_marks->out;
 
-			if (sa_mark.val != 0 && sa_mark.mask != 0) {
+			if (sa_mark.val != 0 && sa_mark.mask != 0 && xfrm_if_id == 0) {
 				struct xfrm_mark xfrm_mark;
 				struct rtattr* mark_attr;
 
@@ -747,12 +747,27 @@ static bool netlink_raw_eroute(const ip_address *this_host,
 			memcpy(RTA_DATA(attr), &xfrm_if_id, sizeof(uint32_t));
 			req.n.nlmsg_len += attr->rta_len;
 
-			/* XFRMA_SET_MARK =  XFRMA_IF_ID/0xffffffff */
-			attr = (struct rtattr *)((char *)&req + req.n.nlmsg_len);
-			attr->rta_type = XFRMA_SET_MARK;
-			attr->rta_len = RTA_LENGTH(sizeof(uint32_t));
-			memcpy(RTA_DATA(attr), &xfrm_if_id, sizeof(uint32_t));
-			req.n.nlmsg_len += attr->rta_len;
+			if (sa_marks->out.val == 0 && sa_marks->out.mask == 0) {
+				/* XFRMA_SET_MARK =  XFRMA_IF_ID/0xffffffff */
+				attr = (struct rtattr *)((char *)&req + req.n.nlmsg_len);
+				attr->rta_type = XFRMA_SET_MARK;
+				attr->rta_len = RTA_LENGTH(sizeof(uint32_t));
+				memcpy(RTA_DATA(attr), &xfrm_if_id, sizeof(uint32_t));
+				req.n.nlmsg_len += attr->rta_len;
+			} else {
+				/* mark-out=mark/mask is configured manually */
+				attr = (struct rtattr *)((char *)&req + req.n.nlmsg_len);
+				attr->rta_type = XFRMA_SET_MARK;
+				attr->rta_len = RTA_LENGTH(sizeof(uint32_t));
+				memcpy(RTA_DATA(attr), &sa_marks->out.val, sizeof(uint32_t));
+				req.n.nlmsg_len += attr->rta_len;
+
+				attr = (struct rtattr *)((char *)&req + req.n.nlmsg_len);
+				attr->rta_type = XFRMA_SET_MARK_MASK;
+				attr->rta_len = RTA_LENGTH(sizeof(uint32_t));
+				memcpy(RTA_DATA(attr), &sa_marks->out.mask, sizeof(uint32_t));
+				req.n.nlmsg_len += attr->rta_len;
+			}
 #endif
 	       }
 
@@ -1561,12 +1576,27 @@ static bool netlink_add_sa(const struct kernel_sa *sa, bool replace)
 		req.n.nlmsg_len += attr->rta_len;
 		attr = (struct rtattr *)((char *)attr + attr->rta_len);
 
-		/* XFRMA_SET_MARK =  XFRMA_IF_ID/0xffffffff */
-		attr->rta_type = XFRMA_SET_MARK;
-		attr->rta_len = RTA_LENGTH(sizeof(uint32_t));
-		memcpy(RTA_DATA(attr), &sa->xfrm_if_id, sizeof(uint32_t));
-		req.n.nlmsg_len += attr->rta_len;
-		attr = (struct rtattr *)((char *)attr + attr->rta_len);
+		if (sa->mark_set.val != 0 || sa->mark_set.mask != 0) {
+			/* mark-out=mark/mask is configured manually */
+			attr->rta_type = XFRMA_SET_MARK;
+			attr->rta_len = RTA_LENGTH(sizeof(uint32_t));
+			memcpy(RTA_DATA(attr), &sa->mark_set.val, sizeof(uint32_t));
+			req.n.nlmsg_len += attr->rta_len;
+			attr = (struct rtattr *)((char *)&req + req.n.nlmsg_len);
+
+			attr->rta_type = XFRMA_SET_MARK_MASK;
+			attr->rta_len = RTA_LENGTH(sizeof(uint32_t));
+			memcpy(RTA_DATA(attr), &sa->mark_set.mask, sizeof(uint32_t));
+			req.n.nlmsg_len += attr->rta_len;
+			attr = (struct rtattr *)((char *)&req + req.n.nlmsg_len);
+		} else {
+			/* XFRMA_SET_MARK =  XFRMA_IF_ID/0xffffffff */
+			attr->rta_type = XFRMA_SET_MARK;
+			attr->rta_len = RTA_LENGTH(sizeof(uint32_t));
+			memcpy(RTA_DATA(attr), &sa->xfrm_if_id, sizeof(uint32_t));
+			req.n.nlmsg_len += attr->rta_len;
+			attr = (struct rtattr *)((char *)attr + attr->rta_len);
+		}
 #endif
 	}
 
-- 
2.21.3



More information about the Swan mailing list