housecarl/scripts/e2e.sh

#!/bin/bash
# End-to-end test with real traffic: a Debian 13 router container (nftables
# 1.1.3) between an "internet" network and a LAN, IPv4 and IPv6, runs
# housecarld. The maintainer's own router ruleset (internal/importer/
# testdata/router.nft, with lab addresses) is loaded as it is today,
# imported with housecarld in shadow mode beside it, taken over, and then
# checked with connections: port forwards, hairpin NAT, the IPv6 pinhole,
# masquerading, antispoofing, the drop log, a change on probation that cuts
# the client off and rolls itself back, a restart and the boot ruleset.
# Needs Docker; no root on the host. Run from the repository root:
#
#   make e2e        or        scripts/e2e.sh
#
# KEEP=1 leaves the containers and networks running. Set DOCKER_HOST if
# Docker isn't at its default socket.
set -u
ROOT=$(cd "$(dirname "$0")/.." && pwd)
P=${E2E_PREFIX:-hce2e}
IMAGE=${E2E_IMAGE:-housecarl-nft}
WORK=${WORK:-$(mktemp -d "${TMPDIR:-/tmp}/housecarl-e2e.XXXXXX")}
docker info >/dev/null 2>&1 || { echo "Docker isn't reachable (set DOCKER_HOST?)"; exit 2; }

(cd "$ROOT" && CGO_ENABLED=0 go build -o "$WORK/housecarld" ./cmd/housecarld && CGO_ENABLED=0 go build -o "$WORK/hc" ./cmd/hc) || exit 1
# The image: Debian 13 with nftables and network tools. 'docker build'
# can't reach the network on some hosts, so it is made with run + commit.
if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
	docker rm -f ${P}-build >/dev/null 2>&1
	docker run --name ${P}-build debian:trixie sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get update -qq && apt-get install -y -qq --no-install-recommends nftables iproute2 iputils-ping netcat-openbsd curl ca-certificates >/dev/null 2>&1 && rm -rf /var/lib/apt/lists/*' || exit 1
	docker commit ${P}-build "$IMAGE" >/dev/null && docker rm ${P}-build >/dev/null || exit 1
fi

cleanup() {
	if [ "${KEEP:-}" = 1 ]; then echo "left running: containers ${P}-*, networks ${P}-*, files in $WORK"; return; fi
	docker exec ${P}-router sh -c 'rm -rf /w/data /w/backups /w/*.log' >/dev/null 2>&1
	docker rm -f $(docker ps -aq --filter "name=^${P}-") >/dev/null 2>&1
	docker network rm ${P}-wan ${P}-lan ${P}-wg >/dev/null 2>&1
	rm -rf "$WORK"
}
trap cleanup EXIT
docker rm -f $(docker ps -aq --filter "name=^${P}-") >/dev/null 2>&1
docker network rm ${P}-wan ${P}-lan ${P}-wg >/dev/null 2>&1

# The lab: "the internet" 198.18.1.0/24 and 2001:db8:1::/64; the LAN
# 10.88.10.0/24 and 2001:db8:7f3d:10::/64 (the /48 is the delegated
# prefix); a WireGuard-like network 10.88.50.0/24. The router has .1 on its
# inside networks (Docker's gateway moves to .254).
docker network create --ipv6 --subnet 198.18.1.0/24 --gateway 198.18.1.254 --subnet 2001:db8:1::/64 --gateway 2001:db8:1::fe ${P}-wan >/dev/null || exit 1
docker network create --ipv6 --subnet 10.88.10.0/24 --gateway 10.88.10.254 --subnet 2001:db8:7f3d:10::/64 --gateway 2001:db8:7f3d:10::fe ${P}-lan >/dev/null || exit 1
docker network create --subnet 10.88.50.0/24 --gateway 10.88.50.254 ${P}-wg >/dev/null || exit 1
WAN4=198.18.1.2 WAN6=2001:db8:1::2 NET4=198.18.1.10 NET6=2001:db8:1::10
LAN4=10.88.10.1 LAN6=2001:db8:7f3d:10::1 JF4=10.88.10.57 JF6=2001:db8:7f3d:10::57 CLIENT4=10.88.10.20

sysctls="--sysctl net.ipv4.ip_forward=1 --sysctl net.ipv6.conf.all.forwarding=1 --sysctl net.ipv6.conf.all.keep_addr_on_down=1 --sysctl net.ipv6.conf.default.keep_addr_on_down=1"
docker run -d --name ${P}-router --hostname router --cap-add NET_ADMIN $sysctls \
	--network ${P}-wan --ip $WAN4 --ip6 $WAN6 -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1
docker network connect --ip $LAN4 --ip6 $LAN6 ${P}-lan ${P}-router || exit 1
docker network connect --ip 10.88.50.1 ${P}-wg ${P}-router || exit 1
# The router's interfaces get the names of the real one: wan, enp2s0, wg0
# (attached in that order: eth0, eth1, eth2).
docker exec ${P}-router sh -c 'for p in eth0:wan eth1:enp2s0 eth2:wg0; do i=${p%:*} n=${p#*:}; ip link set $i down && ip link set $i name $n && ip link set $n up; done
	ip route replace default via 198.18.1.254' || exit 1
# Hosts: one on the internet, a server and a client on the LAN; inside
# hosts route through the router, the internet reaches the delegated
# prefix through it.
host() { # NAME NETWORK IP4 IP6 GATEWAY4 GATEWAY6
	docker run -d --name ${P}-$1 --cap-add NET_ADMIN --sysctl net.ipv6.conf.all.keep_addr_on_down=1 --network ${P}-$2 --ip $3 ${4:+--ip6 $4} -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1
	docker exec ${P}-$1 sh -c "ip route replace default via $5 ${6:+&& ip -6 route replace default via $6}" || exit 1
}
host net wan $NET4 $NET6 198.18.1.254
docker exec ${P}-net ip -6 route add 2001:db8:7f3d::/48 via $WAN6
host jf lan $JF4 $JF6 $LAN4 $LAN6
host client lan $CLIENT4 2001:db8:7f3d:10::20 $LAN4 $LAN6
# Servers: jellyfin on 443 (IPv4 and IPv6) and 22, a listener on the
# internet that logs who connects.
docker exec -d ${P}-jf sh -c 'while true; do echo jellyfin | nc -N -l 0.0.0.0 443; done'
docker exec -d ${P}-jf sh -c 'while true; do echo jellyfin6 | nc -6 -N -l :: 443; done'
docker exec -d ${P}-jf sh -c 'while true; do echo ssh | nc -6 -N -l :: 22; done'
docker exec -d ${P}-net sh -c 'nc -lkvn 0.0.0.0 9000 2>>/w/net.log >/dev/null'

PASS=0; FAIL=0
ok() { echo "PASS: $1"; PASS=$((PASS+1)); }
bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); }
# run DESC EXPECTED_RC CMD...: runs a command, checks its exit code and
# keeps the output in $OUT.
run() {
	local desc=$1 want=$2; shift 2
	OUT=$("$@" 2>&1); local rc=$?
	if [ "$rc" = "$want" ]; then ok "$desc (rc=$rc)"; else bad "$desc: rc=$rc want $want"; echo "$OUT" | sed 's/^/    /'; fi
}
contains() { if echo "$OUT" | grep -q -- "$1"; then ok "output has '$1'"; else bad "output lacks '$1'"; echo "$OUT" | sed 's/^/    /'; fi; }
not_contains() { if echo "$OUT" | grep -q -- "$1"; then bad "output has '$1'"; echo "$OUT" | sed 's/^/    /'; else ok "output lacks '$1'"; fi; }
R() { docker exec ${P}-router "$@"; }
H() { docker exec ${P}-router /w/hc "$@"; }
# connect FROM HOST PORT: what the server says, within 3 seconds.
connect() { docker exec ${P}-$1 nc -w 3 "$2" "$3" </dev/null; }

cat > "$WORK/housecarl.yaml" <<'EOF'
listen: ["0.0.0.0:8070"]
database: /w/data/housecarl.db
ruleset: /w/data/ruleset.nft
backup: {dir: /w/backups}
log: {level: debug}
stats: {database: /w/data/stats.db}
EOF
{ cat "$WORK/housecarl.yaml"; echo "shadow: {enabled: true}"; } > "$WORK/housecarl-shadow.yaml"
CONF=housecarl.yaml
start() { R sh -c "(HOUSECARL_STATS_INTERVAL=3s /w/housecarld -c /w/$CONF serve >> /w/srv.log 2>&1 &); for i in \$(seq 1 30); do [ -S /run/housecarl/housecarl.sock ] && exit 0; sleep 0.3; done; exit 1"; }
stop() { docker exec ${P}-router sh -c 'kill $(pidof housecarld)'; sleep 1; }

# ---- today's router: the hand-written ruleset

sed -e 's/217\.155\.8\.194/198.18.1.2/g' -e 's/10\.0\.10\./10.88.10./g' -e 's/10\.50\.50\./10.88.50./g' \
	-e 's/2a02:8013:7f3d/2001:db8:7f3d/g' "$ROOT/internal/importer/testdata/router.nft" > "$WORK/router.nft"
run "today's ruleset loads" 0 R nft -f /w/router.nft
R nft add table ip other
run "the port forward works under it" 0 connect net $WAN4 8096
contains "jellyfin"

# ---- import in shadow mode: beside today's ruleset, changing nothing

run "init" 0 R sh -c 'echo secret-pass-123 | /w/housecarld -c /w/housecarl.yaml init --password-stdin'
CONF=housecarl-shadow.yaml
run "start housecarld in shadow mode" 0 start
run "status" 0 H status
contains "Mode:.*SHADOW"
contains "Other tables:.*inet main"
run "two tables: say which" 6 H import --defines /w/router.nft -n
contains "several tables"
run "dry run of the import" 0 H import --table "inet main" --defines /w/router.nft -n
contains "5 port forwards, 2 source NAT rules"
contains "dry run: nothing was changed"
contains "dhcp (enp2s0: the define dhcp_int)"
run "zone names given at import" 0 H import --table "inet main" --defines /w/router.nft --zone enp2s0=lan --zone wg0=vpn -n
contains "lan (enp2s0: as given)"
contains "vpn (wg0: as given)"
run "import" 0 H import --table "inet main" --defines /w/router.nft
contains "applied and saved"
run "the shadow ruleset is in force" 0 R nft list table inet housecarl
contains "would dnat\] jellyfin"
contains "priority filter + 10; policy accept;"
not_contains "dnat ip to"
not_contains "maxseg size set"
not_contains "flowtable"
run "nothing in it drops, rejects or translates" 1 R sh -c "nft list table inet housecarl | grep -v 'log prefix' | sed 's/comment.*//; s/ct status dnat//' | grep -wE 'drop|reject|dnat|snat|masquerade|redirect'"
run "no warning about today's ruleset beside it" 0 H status
not_contains "takeover_pending"
not_contains "shadow_unprotected"
run "what it loads once shadow mode is off" 0 H ruleset --enforcing
contains "dnat ip to 10.88.10.57:443"
run "the port forward works, by today's ruleset" 0 connect net $WAN4 8096
contains "jellyfin"
run "and Housecarl's would-be DNAT counted it" 0 R nft list chain inet housecarl dstnat
contains 'packets [1-9][0-9]* bytes [0-9]* comment "hc:f[0-9]* \[would dnat\] jellyfin'
# A drop Housecarl has and today's ruleset doesn't: logged, let through.
SR=$(H -o id rule add forward --from $CLIENT4 --service tcp/9000 -a drop --log --first --comment "e2e shadow")
: > "$WORK/net.log"
run "a shadow drop lets the connection through" 0 docker exec ${P}-client sh -c "echo hi | nc -N -w 3 $NET4 9000"
sleep 1
run "it reached the internet" 0 grep -q "$WAN4" "$WORK/net.log"
run "the log says it would have been dropped" 0 H log
contains "would drop, let through"
run "the shadow rule goes" 0 H rule delete -y "$SR"
run "no takeover in shadow mode" 5 H takeover "inet main" -y
contains "shadow mode"
run "healthz" 0 R curl -sf http://127.0.0.1:8070/healthz
contains '"mode":"shadow"'
run "the boot ruleset is the shadow one" 0 R grep -q "would dnat" /w/data/ruleset.nft
run "the boot fallback filters nothing" 1 R grep -wE "drop|masquerade" /w/data/ruleset-fallback.nft
# Today's ruleset starts with flush ruleset: reloading it deletes
# Housecarl's table, which housecarld puts back.
run "today's ruleset reloaded" 0 R nft -f /w/router.nft
R nft add table ip other
sleep 4
run "Housecarl's table is back" 0 R nft list table inet housecarl
contains "would dnat"
run "and logged" 0 R grep -q "put it back" /w/srv.log

# ---- enforcing, and take over

stop
CONF=housecarl.yaml
run "start housecarld enforcing" 0 start
run "status" 0 H status
not_contains "SHADOW"
run "the ruleset is in force" 0 R nft list table inet housecarl
contains "dnat ip to 10.88.10.57:443"
not_contains "would"
run "zones" 0 H zone list
contains "wan.*wan.*yes"
contains "dhcp.*enp2s0"
run "port forwards" 0 H forward list
contains "jellyfin.*198.18.1.2 tcp/8096.*10.88.10.57:443"
run "take over the old table" 0 H takeover "inet main" -y
run "only Housecarl's table and the other one remain" 0 R nft list tables
not_contains "inet main"
contains "ip other"

# ---- traffic

run "port forward from the internet" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "hairpin: the public address from the LAN" 0 connect client $WAN4 8096
contains "^jellyfin$"
# The IPv6 rule stays a forward rule: jellyfin and haproxy both use inside
# port 443, so the importer doesn't guess whose pinhole it is.
run "the IPv6 rule" 0 H rule list forward
contains "to 2001:db8:7f3d:10::57  tcp/443  *accept"
run "IPv6 to the host's port 443 from the internet" 0 connect net $JF6 443
contains "jellyfin6"
run "IPv6 to another port of the host is dropped" 1 connect net $JF6 22
run "masquerade: the LAN reaches the internet as the router" 0 docker exec ${P}-client sh -c "echo hi | nc -N -w 3 $NET4 9000"
sleep 1
run "the internet saw the router's address" 0 cat "$WORK/net.log"
contains "$WAN4"
not_contains "$CLIENT4"
run "nothing else gets in from the internet" 1 connect net $JF4 443
# Antispoofing: a source that can't come from the internet.
docker exec ${P}-net ip addr add 10.9.9.9/32 dev eth0
docker exec ${P}-net ping -c 2 -W 1 -I 10.9.9.9 $WAN4 >/dev/null 2>&1
run "counters" 0 H counters
contains "b-antispoof  *[1-9]"

# ---- the API from the LAN, on probation

# Allow the API from the trusted networks (through the local socket: no
# probation).
APIRULE=$(H -o id rule add input --from trusted_ips --service tcp/8070 --comment "Housecarl API")
run "the API rule was added" 0 test -n "$APIRULE"
run "probation of 10 seconds" 0 H settings set --probation 10s
TOKEN=$(R /w/housecarld -c /w/housecarl.yaml user token admin e2e 2>/dev/null | tail -1)
C() { docker exec ${P}-client /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" "$@"; }
run "the API from the LAN" 0 C whoami
run "a change over the network is on probation" 0 C --no-confirm zone add guests --interface guest0
contains "applied on probation"
run "confirmed" 0 C confirm
contains "confirmed 1 change;"
run "a change that cuts the client off" 0 C --no-confirm rule disable "$APIRULE"
contains "on probation"
run "the client is cut off (a timeout)" 124 docker exec ${P}-client timeout 3 /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" whoami
sleep 12
run "rolled back by itself: the client is back" 0 C whoami
run "the rule is enabled again" 0 H rule show "$APIRULE"
not_contains "disabled"
run "the rollback is in the history" 0 H history list
contains "rollback"
# Confirmed by the client itself: it reached the API over a new connection.
run "a change confirmed by its client" 0 C zone add guests2 --interface guest2
contains "applied and confirmed"
run "nothing waits" 0 C status
not_contains "Probation:"
run "the confirmation is audited" 0 H audit --action config.confirm
contains "confirmed automatically"
# A change that cuts its client off can't be confirmed by it: rolled back.
run "a lockout change, not confirmed" 0 C rule disable "$APIRULE"
contains "couldn't confirm it"
sleep 3
run "rolled back: the client is back" 0 C whoami
run "the rule is enabled again" 0 H rule show "$APIRULE"
not_contains "disabled"
run "a person must confirm, by setting" 0 H settings set --auto-confirm off
run "then a change waits" 0 C zone add guests3 --interface guest3
contains "applied on probation"
run "confirmed by a person" 0 C confirm
run "back to automatic" 0 H settings set --auto-confirm on

# ---- changesets: staged, reviewed, applied together

CS=$(C changeset create "e2e batch" -q)
run "a changeset" 0 test -n "$CS"
CE() { docker exec -e HOUSECARL_CHANGESET=$CS ${P}-client /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" "$@"; }
run "staged from HOUSECARL_CHANGESET" 0 CE set add e2e_block 198.51.100.7
contains "staged in changeset $CS as operation 1"
contains "from HOUSECARL_CHANGESET"
run "a rule naming the staged set" 0 CE rule add input --from e2e_block --service ssh -a drop
contains "operation 2"
run "disabling the jellyfin forward, staged" 0 CE forward update jellyfin --disabled
contains "operation 3"
run "nothing is live yet: the forward still works" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "the set isn't there yet" 4 H set show e2e_block
run "the changeset's diff" 0 CE changeset diff
contains "^+.*ip saddr @e2e_block_v4 tcp dport 22 counter drop"
contains "^-.*dnat ip to 10.88.10.57:443"
contains "valid: nftables accepts"
run "remove the forward operation" 0 CE changeset remove 3
contains "has 2 operations"
run "the diff without it" 0 CE changeset diff
not_contains "dnat ip to 10.88.10.57"
run "applied together, and confirmed by this client" 0 CE changeset apply
contains "applied and confirmed"
contains "HOUSECARL_CHANGESET still names changeset $CS"
run "one history entry" 0 H history list
contains "e2e batch"
run "confirmed" 0 C confirm
run "the set is there now" 0 H set show e2e_block
run "the forward still works" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "a stale HOUSECARL_CHANGESET is refused" 1 CE zone add stale -i stale0
contains "is applied"
CS2=$(H changeset create -q)
UNDO=$(H zone add undome -i undo0 | sed -n 's/.*history \([0-9]*\);.*/\1/p')
run "an undo is staged" 0 H -C "$CS2" history undo "$UNDO"
contains "staged in changeset $CS2"
run "not applied yet: the zone is there" 0 H zone show undome
run "an import can't be staged" 6 sh -c "echo '{\"nftables\":[]}' | docker exec -i ${P}-router /w/hc -C $CS2 import -"
contains "can't be staged"
run "discarded" 0 H changeset discard "$CS2"
run "the zone stays" 0 H zone show undome
run "a dry run shows the ruleset's diff" 0 H -n set add e2e_dry 10.1.2.3
contains "^+.*set e2e_dry_v4"

# ---- quotas: an allowance per device, kept across a restart, reset

QR=$(H -o id rule add forward --from $CLIENT4 --service tcp/9000 --quota 1mbytes,over,per-source -a drop --first --comment "e2e quota")
run "a per-device quota" 0 test -n "$QR"
send() { docker exec ${P}-client sh -c "head -c $1 /dev/zero | nc -N -w 3 $NET4 9000"; }
run "within the quota" 0 send 300000
send 2000000 >/dev/null 2>&1
run "used up: the device is cut off" 1 send 1000
run "the usage" 0 H quotas
contains "e2e quota.*1.0 MB each.*by hand"
contains "$CLIENT4  *1\.[0-9] MB"
stop; start
run "after a restart, still used up" 1 send 1000
run "the usage after a restart" 0 H quotas
contains "$CLIENT4  *1\.[0-9] MB"
run "reset by hand" 0 H rule reset-quota "$QR"
run "the device is back" 0 send 1000
run "the reset is audited" 0 H audit --action quota.
contains "quota.reset"
run "the quota rule goes" 0 H rule delete -y "$QR"

# ---- the zone matrix, of the imported ruleset

run "the matrix" 0 H matrix
contains "^firewall  *allow"
contains "^dhcp  *partly +[0-9]"
contains "dhcp → wan  *.*from trusted_ips"
run "LAN to the internet" 0 H matrix dhcp wan
contains "new connections: partly allowed"
contains "allowed  *from trusted_ips  *rule forward #[0-9]* (Forward from trusted IPs)  *[0-9]"
contains "blocked  *the rest  *forward policy  *[0-9]"
not_contains "wg0_net"
run "the internet to the firewall" 0 H matrix wan firewall
contains "blocked  *sources that can't come from outside"
contains "allowed  *udp/51820"
contains "allowed  *raw: icmpv6 type"

# ---- statistics (sampled every 3 seconds here)

run "traffic per device on" 0 H settings set --stats-per-device on --log-drops on
run "the device counters are in the ruleset" 0 R nft list set inet housecarl stats_up_v4
send 500000 >/dev/null 2>&1
run "a connection from the internet, dropped and logged" 1 connect net $WAN4 23
sleep 8
run "statistics are collected" 0 H stats status
contains "Collecting:  *yes"
contains "Per device:  *on"
run "the summary" 0 H stats --since 1h
contains "^wan  *[0-9.]* [KM]B"
contains "Dropped:  *[1-9]"
run "traffic per zone over time" 0 H stats traffic --since 1h --step 1m
contains "wan ↓"
run "the devices" 0 H stats top devices --since 1h
contains "^$CLIENT4  *[0-9.]* KB  *[0-9.]* KB"
run "the sources blocked" 0 H stats top blocked --since 1h
contains "^$NET4  *[1-9]"
run "what they tried" 0 H stats top tried --since 1h
contains "^tcp/23"
run "hits per rule" 0 H stats rules --since 1h
contains "input policy  *[1-9]"
contains "rule #[0-9]* (deleted)  *[1-9]"
run "connections over time" 0 H stats connections --since 1h --step 1m
contains "^[0-9-]* [0-9:]*  *[0-9]*  *[0-9]*"

# ---- connections and tracing

# A connection that stays open, masqueraded on its way out.
docker exec -d ${P}-client sh -c "sleep 30 | nc $NET4 9000"
sleep 1
run "the connection is tracked" 0 H connections --search "$CLIENT4 :9000"
contains "$CLIENT4:[0-9]*  *$NET4:9000  *as $WAN4"
run "closed" 0 H connections close --search "$CLIENT4 :9000" -y
contains "closed [1-9][0-9]* connection"
run "gone" 0 H connections --search "$CLIENT4 :9000"
not_contains ":9000"
# A block added while traffic flows: it stops new connections, not this
# one, unless it checks every packet.
docker exec -d ${P}-client sh -c "rm -f /tmp/ncdone; yes | nc $NET4 9000; touch /tmp/ncdone"
sleep 1
BR=$(H -o id rule add forward --from $CLIENT4 --service tcp/9000 -a reject --reject-with tcp-reset --comment "e2e block" --first)
run "a block for new connections" 0 test -n "$BR"
sleep 2
run "the open upload carries on" 1 docker exec ${P}-client test -f /tmp/ncdone
run "the block checks every packet now" 0 H rule update "$BR" --every-packet
sleep 2
run "the upload is cut" 0 docker exec ${P}-client test -f /tmp/ncdone
# Tracing a new connection: dropped, by that rule.
docker exec -d ${P}-router sh -c "/w/hc trace --address $CLIENT4 --port 9000 --seconds 6 > /w/trace.out 2>&1"
sleep 2
connect client $NET4 9000 >/dev/null 2>&1
sleep 6
run "the trace" 0 cat "$WORK/trace.out"
contains "$CLIENT4:[0-9]* -> $NET4:9000  tcp syn"
contains "inet housecarl forward: rule forward #$BR (e2e block) -> drop"
contains "=> DROPPED by rule forward #$BR (e2e block)"
run "no trace table is left" 0 R nft list tables
not_contains "housecarl-trace"
run "the block goes" 0 H rule delete -y "$BR"

# ---- the drop log

run "log drops from the internet to the API" 0 H rule add input --in wan --service tcp/8070 -a drop --log --first
connect net $WAN4 8070 >/dev/null 2>&1
sleep 1
run "the log shows it" 0 H log
contains "hc input #.* drop.*198.18.1.10:.*198.18.1.2:8070"

# ---- the flowtable, and devices that appear later

run "a flowtable device that doesn't exist yet" 0 H settings set --flowtable wan --flowtable enp2s0 --flowtable hcft0
contains "flowtable_device_missing"
run "the flowtable has the devices that exist" 0 R nft list flowtables
contains "devices = { enp2s0, wan }"
run "forwarded through the flowtable" 0 connect net $WAN4 8096
contains "jellyfin"
run "the device appears" 0 R ip link add hcft0 type veth peer name hcft1
R ip link set hcft0 up
sleep 4
run "applied again with it" 0 R nft list flowtables
contains "hcft0"
run "logged" 0 R grep -c "interfaces changed: applied the ruleset again" /w/srv.log
run "it comes back as a new interface" 0 R sh -c 'ip link del hcft0 && ip link add hcft0 type veth peer name hcft1 && ip link set hcft0 up'
sleep 4
run "applied again" 0 R sh -c 'grep -c "interfaces changed: applied the ruleset again" /w/srv.log | grep -qx 2'
run "the flowtable still has it" 0 R nft list flowtables
contains "hcft0"

# ---- VLAN sub-interfaces: tagged traffic between the router and the client

R sh -c 'ip link add link enp2s0 name enp2s0.30 type vlan id 30 && ip addr add 10.88.30.1/24 dev enp2s0.30 && ip link set enp2s0.30 up'
docker exec ${P}-client sh -c 'ip link add link eth0 name eth0.30 type vlan id 30 && ip addr add 10.88.30.20/24 dev eth0.30 && ip link set eth0.30 up'
run "a zone of a VLAN" 0 H zone add iot --interface enp2s0.30
run "nothing from the VLAN to the API yet" 1 connect client 10.88.30.1 8070
VR=$(H -o id rule add input --in iot --service tcp/8070 --comment "e2e vlan")
run "a rule for the VLAN's zone" 0 test -n "$VR"
run "the VLAN reaches the API" 0 connect client 10.88.30.1 8070
run "the rule counted it" 0 H rule list input
contains "e2e vlan"
run "the flowtable on the VLAN" 0 H settings set --flowtable wan --flowtable enp2s0 --flowtable hcft0 --flowtable enp2s0.30
run "attached" 0 R nft list flowtables
contains "enp2s0.30"
run "the VLAN rule goes" 0 H rule delete -y "$VR"
run "the VLAN zone goes" 0 H zone delete -y iot
R ip link del enp2s0.30

# ---- renaming a zone others refer to

run "rename the imported dhcp zone" 0 H zone update dhcp --name lan
run "its rules follow" 0 H rule list input
contains "lan"
not_contains "dhcp "
run "the ruleset is unchanged by it" 0 R nft list chain inet housecarl input
contains 'iifname "enp2s0" udp dport 67'

# ---- restart and boot

stop
run "the ruleset stays without housecarld" 0 connect net $WAN4 8096
contains "jellyfin"
run "restart housecarld" 0 start
run "after the restart" 0 connect client $WAN4 8096
contains "jellyfin"
run "the boot ruleset" 0 R test -s /w/data/ruleset.nft
R nft flush ruleset
run "nothing forwarded after a flush" 1 connect net $WAN4 8096
run "the boot unit's command restores it" 0 R nft -f /w/data/ruleset.nft
run "forwarded again" 0 connect net $WAN4 8096
contains "jellyfin"
run "the boot ruleset has no flowtable (its devices may not exist yet)" 1 R grep -q flowtable /w/data/ruleset.nft
run "healthz" 0 R curl -sf http://127.0.0.1:8070/healthz
contains '"firewall":"applied"'

# ---- the boot fallback: what the boot unit runs, with the lab's paths

UNIT=$(sed -n 's/^ExecStart=//p' "$ROOT/deploy/systemd/housecarl-ruleset.service" | sed 's|/var/lib/housecarl|/w/data|g; s|/usr/sbin/nft|nft|g')
run "the fallback was saved" 0 R test -s /w/data/ruleset-fallback.nft
R cp /w/data/ruleset.nft /w/ruleset.good
R sh -c 'echo "this is not nftables" >> /w/data/ruleset.nft'
R nft flush ruleset
run "the boot unit fails, and loads the fallback" 1 R sh -c "$UNIT"
contains "the saved ruleset failed to load"
run "the fallback is in force" 0 R nft list table inet housecarl
contains "hc:fallback-input"
run "nothing new gets in from the internet" 1 connect net $WAN4 8096
run "not to the firewall either" 1 connect net $WAN4 8070
: > "$WORK/net.log"
run "the LAN still reaches the internet" 0 connect client $NET4 9000
sleep 1
run "masqueraded" 0 grep -q "$WAN4" "$WORK/net.log"
run "healthz says so" 0 R sh -c 'curl -s -w " %{http_code}" http://127.0.0.1:8070/healthz'
contains '"firewall":"fallback"'
contains " 503"
run "status says so" 0 H status
contains "NOT in force: the boot fallback"
stop
run "housecarld starts" 0 start
run "and puts the real ruleset back" 0 connect net $WAN4 8096
contains "jellyfin"
run "and saves a good boot ruleset" 0 R nft -c -f /w/data/ruleset.nft
run "healthz is happy again" 0 R curl -sf http://127.0.0.1:8070/healthz
contains '"firewall":"applied"'

# ---- backup

run "backup" 0 R /w/housecarld -c /w/housecarl.yaml backup
contains "/w/backups/housecarl-"

echo
echo "RESULT: $PASS passed, $FAIL failed"
[ $FAIL = 0 ]