I'm building a test VPN setup using two libvirt VMs with StrongSwan IKEv2 but can't get traffic to forward. I can see the client sending traffic to the server using tcpdump but the server doesn't do anything with it.
To rule out completely broken configuration, I launched an EC2 test instance. All VMs and EC2 instance are running Rocky Linux 9.3. I ran the same Ansible playbook I'm using for the local server VM on the EC2 test instance. The EC2 instance was able to forward traffic with no issue.
Why doesn't this work?
server interfaces:
[root@server ~] # ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp1s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 52:54:00:ab:6d:28 brd ff:ff:ff:ff:ff:ff
inet 192.168.124.46/24 brd 192.168.124.255 scope global dynamic noprefixroute enp1s0
valid_lft 2044sec preferred_lft 2044sec
inet6 fe80::5054:ff:feab:6d28/64 scope link noprefixroute
valid_lft forever preferred_lft forever
server iptables-save output:
[root@server ~] # iptables-save
# Generated by iptables-save v1.8.8 (nf_tables) on Fri Feb 9 11:24:48 2024
*mangle
:PREROUTING ACCEPT [628:1708335]
:INPUT ACCEPT [594:1704345]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [566:81987]
:POSTROUTING ACCEPT [566:81987]
-A FORWARD -s 172.16.0.0/24 -o enp1s0 -p tcp -m policy --dir in --pol ipsec -m tcp --tcp-flags SYN,RST SYN -m tcpmss --mss 1361:1536 -j TCPMSS --set-mss 1360
COMMIT
# Completed on Fri Feb 9 11:24:48 2024
# Generated by iptables-save v1.8.8 (nf_tables) on Fri Feb 9 11:24:48 2024
*raw
:PREROUTING ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
COMMIT
# Completed on Fri Feb 9 11:24:48 2024
# Generated by iptables-save v1.8.8 (nf_tables) on Fri Feb 9 11:24:48 2024
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [566:81987]
-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-A INPUT -p tcp -m tcp --dport 22 -j ACCEPT
-A INPUT -i lo -j ACCEPT
-A INPUT -p udp -m udp --dport 500 -j ACCEPT
-A INPUT -p udp -m udp --dport 4500 -j ACCEPT
-A INPUT -j DROP
-A FORWARD -s 172.16.0.0/24 -m policy --dir in --pol ipsec --proto esp -j ACCEPT
-A FORWARD -d 172.16.0.0/24 -m policy --dir out --pol ipsec --proto esp -j ACCEPT
-A FORWARD -j DROP
COMMIT
# Completed on Fri Feb 9 11:24:48 2024
# Generated by iptables-save v1.8.8 (nf_tables) on Fri Feb 9 11:24:48 2024
*nat
:PREROUTING ACCEPT [66:14106]
:INPUT ACCEPT [22:8852]
:OUTPUT ACCEPT [4:304]
:POSTROUTING ACCEPT [4:304]
-A POSTROUTING -s 172.16.0.0/24 -o enp1s0 -m policy --dir out --pol ipsec -j ACCEPT
-A POSTROUTING -s 172.16.0.0/24 -o enp1s0 -j MASQUERADE
COMMIT
# Completed on Fri Feb 9 11:24:48 2024
client interfaces during VPN session:
[root@client ~]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp1s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 52:54:00:5b:1a:e1 brd ff:ff:ff:ff:ff:ff
inet 192.168.124.217/24 brd 192.168.124.255 scope global dynamic noprefixroute enp1s0
valid_lft 2084sec preferred_lft 2084sec
inet 172.16.0.2/32 scope global enp1s0
valid_lft forever preferred_lft forever
inet6 fe80::5054:ff:fe5b:1ae1/64 scope link noprefixroute
valid_lft forever preferred_lft forever
tcpdump timeline from server (192.168.124.46):
(on client, run `charon-cmd --host 192.168.124.46 --identity dave --cert server.pem`; connects successfully)
11:16:17.565576 IP 192.168.124.217.35967 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: parent_sa ikev2_init[I]
11:16:17.565934 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.35967: NONESP-encap: isakmp: parent_sa ikev2_init[R]
11:16:17.566461 IP 192.168.124.217.35967 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: parent_sa ikev2_init[I]
11:16:17.566740 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.35967: NONESP-encap: isakmp: parent_sa ikev2_init[R]
11:16:17.567137 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa ikev2_auth[I]
11:16:17.567441 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa ikev2_auth[R]
11:16:17.568100 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa ikev2_auth[I]
11:16:17.568167 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa ikev2_auth[R]
11:16:18.921417 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa ikev2_auth[I]
11:16:18.921651 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa ikev2_auth[R]
11:16:18.921959 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa ikev2_auth[I]
11:16:18.922016 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa ikev2_auth[R]
11:16:18.922235 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa ikev2_auth[I]
11:16:18.923167 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa ikev2_auth[R]
(on client, run `dig @1.1.1.1 CH TXT whoami.cloudflare.`; request just times out)
11:16:24.017772 IP 192.168.124.217 > 192.168.124.46: ESP(spi=0xc2944693,seq=0x1), length 120
11:16:29.017212 IP 192.168.124.217 > 192.168.124.46: ESP(spi=0xc2944693,seq=0x2), length 120
(on client, terminate charon-cmd)
11:16:31.240008 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa inf2[I]
11:16:31.240907 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa inf2[R]
tcpdump timeline from client (192.168.124.217):
(on client, run `charon-cmd --host 192.168.124.46 --identity dave --cert server.pem`; connects successfully)
11:16:17.563485 IP 192.168.124.217.35967 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: parent_sa ikev2_init[I]
11:16:17.563924 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.35967: NONESP-encap: isakmp: parent_sa ikev2_init[R]
11:16:17.564405 IP 192.168.124.217.35967 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: parent_sa ikev2_init[I]
11:16:17.564723 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.35967: NONESP-encap: isakmp: parent_sa ikev2_init[R]
11:16:17.565084 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa ikev2_auth[I]
11:16:17.565427 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa ikev2_auth[R]
11:16:17.566047 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa ikev2_auth[I]
11:16:17.566150 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa ikev2_auth[R]
11:16:18.919321 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa ikev2_auth[I]
11:16:18.919649 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa ikev2_auth[R]
11:16:18.919898 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa ikev2_auth[I]
11:16:18.920004 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa ikev2_auth[R]
11:16:18.920184 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa ikev2_auth[I]
11:16:18.921151 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa ikev2_auth[R]
11:16:19.033177 IP 192.168.124.217.mdns > 224.0.0.251.mdns: 0 [2q] [2n] ANY (QM)? 2.0.16.172.in-addr.arpa. ANY (QM)? client.local. (89)
11:16:19.283570 IP 192.168.124.217.mdns > 224.0.0.251.mdns: 0 [2q] [2n] ANY (QM)? 2.0.16.172.in-addr.arpa. ANY (QM)? client.local. (89)
11:16:19.534004 IP 192.168.124.217.mdns > 224.0.0.251.mdns: 0 [2q] [2n] ANY (QM)? 2.0.16.172.in-addr.arpa. ANY (QM)? client.local. (89)
11:16:19.734337 IP 192.168.124.217.mdns > 224.0.0.251.mdns: 0*- [0q] 3/0/0 (Cache flush) PTR client.local., (Cache flush) A 172.16.0.2, (Cache flush) A 192.168.124.217 (93)
11:16:20.795580 IP 192.168.124.217.mdns > 224.0.0.251.mdns: 0*- [0q] 3/0/0 (Cache flush) PTR client.local., (Cache flush) A 172.16.0.2, (Cache flush) A 192.168.124.217 (93)
(on client, run `dig @1.1.1.1 CH TXT whoami.cloudflare.`; request just times out)
11:16:22.857845 IP 192.168.124.217.mdns > 224.0.0.251.mdns: 0*- [0q] 3/0/0 (Cache flush) PTR client.local., (Cache flush) A 172.16.0.2, (Cache flush) A 192.168.124.217 (93)
11:16:24.015596 IP 192.168.124.217 > 192.168.124.46: ESP(spi=0xc2944693,seq=0x1), length 120
11:16:29.015083 IP 192.168.124.217 > 192.168.124.46: ESP(spi=0xc2944693,seq=0x2), length 120
(on client, terminate charon-cmd)
11:16:31.237922 IP 192.168.124.217.52793 > 192.168.124.46.ipsec-nat-t: NONESP-encap: isakmp: child_sa inf2[I]
11:16:31.238903 IP 192.168.124.46.ipsec-nat-t > 192.168.124.217.52793: NONESP-encap: isakmp: child_sa inf2[R]
11:16:31.238935 IP 192.168.124.217.mdns > 224.0.0.251.mdns: 0*- [0q] 2/0/0 (Cache flush) PTR client.local., (Cache flush) A 172.16.0.2 (77)
(on client, retry dig command without vpn connected)
11:16:40.086784 IP 192.168.124.217.36140 > 1.1.1.1.domain: 2399+ [1au] TXT CHAOS? whoami.cloudflare. (58)
11:16:40.103724 IP 1.1.1.1.domain > 192.168.124.217.36140: 2399 1/0/1 CHAOS TXT "72.131.108.98" (72)
