adam/housecarl
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity
Files
1fcc204ff82def6e6780520d2e54135bf51ec391
housecarl/scripts/e2e.sh
T
abrown38andClaude Opus 5.5 68ba2d06c5 Sets from Jarl: sets holding the addresses of names in Jarl 
A set can take its addresses from Jarl: every A and AAAA record of its
names (NAME@VIEW), CNAMEs followed within Jarl's zones in the same view.
housecarld reads them every jarl.refresh and after each change, keeps
only the addresses within the zones and networks the set allows, and
keeps what it last read (saved in the database and the boot ruleset)
while Jarl can't be reached. A change of the addresses applies the
ruleset again without a history entry and is audited (set.jarl).

hc set add --jarl --within, hc set jarl; the kind "from Jarl" and
Addresses… on the Sets page; GET /sets/{name}/jarl and /jarl/refresh;
validation codes set_jarl_*, status warnings jarl_*; docs/jarl.md.
e2e phase with scripts/fakejarl.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-23 13:51:55 +01:00

711 lines
35 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# End-to-end test with real traffic: a Debian 13 router container (nftables
# 1.1.3) between an "internet" network and a LAN, IPv4 and IPv6, runs
# housecarld. The maintainer's own router ruleset (internal/importer/
# testdata/router.nft, with lab addresses) is loaded as it is today,
# imported with housecarld in shadow mode beside it, taken over, and then
# checked with connections: port forwards, hairpin NAT, the IPv6 pinhole,
# masquerading, antispoofing, the drop log, a change on probation that cuts
# the client off and rolls itself back, a restart and the boot ruleset.
# Needs Docker; no root on the host. Run from the repository root:
#
# make e2e or scripts/e2e.sh
#
# KEEP=1 leaves the containers and networks running. Set DOCKER_HOST if
# Docker isn't at its default socket.
set -u
ROOT=$(cd "$(dirname "$0")/.." && pwd)
P=${E2E_PREFIX:-hce2e}
IMAGE=${E2E_IMAGE:-housecarl-nft}
WORK=${WORK:-$(mktemp -d "${TMPDIR:-/tmp}/housecarl-e2e.XXXXXX")}
docker info >/dev/null 2>&1 || { echo "Docker isn't reachable (set DOCKER_HOST?)"; exit 2; }
(cd "$ROOT" && CGO_ENABLED=0 go build -o "$WORK/housecarld" ./cmd/housecarld && CGO_ENABLED=0 go build -o "$WORK/hc" ./cmd/hc &&
CGO_ENABLED=0 go build -o "$WORK/fakejarl" ./scripts/fakejarl) || exit 1
# The image: Debian 13 with nftables and network tools. 'docker build'
# can't reach the network on some hosts, so it is made with run + commit.
if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
docker rm -f ${P}-build >/dev/null 2>&1
docker run --name ${P}-build debian:trixie sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get update -qq && apt-get install -y -qq --no-install-recommends nftables iproute2 iputils-ping netcat-openbsd curl ca-certificates >/dev/null 2>&1 && rm -rf /var/lib/apt/lists/*' || exit 1
docker commit ${P}-build "$IMAGE" >/dev/null && docker rm ${P}-build >/dev/null || exit 1
fi
cleanup() {
if [ "${KEEP:-}" = 1 ]; then echo "left running: containers ${P}-*, networks ${P}-*, files in $WORK"; return; fi
docker exec ${P}-router sh -c 'rm -rf /w/data /w/backups /w/*.log' >/dev/null 2>&1
docker rm -f $(docker ps -aq --filter "name=^${P}-") >/dev/null 2>&1
docker network rm ${P}-wan ${P}-lan ${P}-wg >/dev/null 2>&1
rm -rf "$WORK"
}
trap cleanup EXIT
docker rm -f $(docker ps -aq --filter "name=^${P}-") >/dev/null 2>&1
docker network rm ${P}-wan ${P}-lan ${P}-wg >/dev/null 2>&1
# The lab: "the internet" 198.18.1.0/24 and 2001:db8:1::/64; the LAN
# 10.88.10.0/24 and 2001:db8:7f3d:10::/64 (the /48 is the delegated
# prefix); a WireGuard-like network 10.88.50.0/24. The router has .1 on its
# inside networks (Docker's gateway moves to .254).
docker network create --ipv6 --subnet 198.18.1.0/24 --gateway 198.18.1.254 --subnet 2001:db8:1::/64 --gateway 2001:db8:1::fe ${P}-wan >/dev/null || exit 1
docker network create --ipv6 --subnet 10.88.10.0/24 --gateway 10.88.10.254 --subnet 2001:db8:7f3d:10::/64 --gateway 2001:db8:7f3d:10::fe ${P}-lan >/dev/null || exit 1
docker network create --subnet 10.88.50.0/24 --gateway 10.88.50.254 ${P}-wg >/dev/null || exit 1
WAN4=198.18.1.2 WAN6=2001:db8:1::2 NET4=198.18.1.10 NET6=2001:db8:1::10
LAN4=10.88.10.1 LAN6=2001:db8:7f3d:10::1 JF4=10.88.10.57 JF6=2001:db8:7f3d:10::57 CLIENT4=10.88.10.20
sysctls="--sysctl net.ipv4.ip_forward=1 --sysctl net.ipv6.conf.all.forwarding=1 --sysctl net.ipv6.conf.all.keep_addr_on_down=1 --sysctl net.ipv6.conf.default.keep_addr_on_down=1"
docker run -d --name ${P}-router --hostname router --cap-add NET_ADMIN $sysctls \
--network ${P}-wan --ip $WAN4 --ip6 $WAN6 -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1
docker network connect --ip $LAN4 --ip6 $LAN6 ${P}-lan ${P}-router || exit 1
docker network connect --ip 10.88.50.1 ${P}-wg ${P}-router || exit 1
# The client has a name for housecarld's reverse lookups (/etc/hosts
# here, a PTR record in real life); the addresses outside get none.
docker exec ${P}-router sh -c "echo '$CLIENT4 e2e-client.lab' >> /etc/hosts" || exit 1
# The router's interfaces get the names of the real one: wan, enp2s0, wg0
# (attached in that order: eth0, eth1, eth2).
docker exec ${P}-router sh -c 'for p in eth0:wan eth1:enp2s0 eth2:wg0; do i=${p%:*} n=${p#*:}; ip link set $i down && ip link set $i name $n && ip link set $n up; done
ip route replace default via 198.18.1.254' || exit 1
# Hosts: one on the internet, a server and a client on the LAN; inside
# hosts route through the router, the internet reaches the delegated
# prefix through it.
host() { # NAME NETWORK IP4 IP6 GATEWAY4 GATEWAY6
docker run -d --name ${P}-$1 --cap-add NET_ADMIN --sysctl net.ipv6.conf.all.keep_addr_on_down=1 --network ${P}-$2 --ip $3 ${4:+--ip6 $4} -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1
docker exec ${P}-$1 sh -c "ip route replace default via $5 ${6:+&& ip -6 route replace default via $6}" || exit 1
}
host net wan $NET4 $NET6 198.18.1.254
docker exec ${P}-net ip -6 route add 2001:db8:7f3d::/48 via $WAN6
host jf lan $JF4 $JF6 $LAN4 $LAN6
host client lan $CLIENT4 2001:db8:7f3d:10::20 $LAN4 $LAN6
# Servers: jellyfin on 443 (IPv4 and IPv6) and 22, a listener on the
# internet that logs who connects.
docker exec -d ${P}-jf sh -c 'while true; do echo jellyfin | nc -N -l 0.0.0.0 443; done'
docker exec -d ${P}-jf sh -c 'while true; do echo jellyfin6 | nc -6 -N -l :: 443; done'
docker exec -d ${P}-jf sh -c 'while true; do echo ssh | nc -6 -N -l :: 22; done'
docker exec -d ${P}-net sh -c 'nc -lkvn 0.0.0.0 9000 2>>/w/net.log >/dev/null'
PASS=0; FAIL=0
ok() { echo "PASS: $1"; PASS=$((PASS+1)); }
bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); }
# run DESC EXPECTED_RC CMD...: runs a command, checks its exit code and
# keeps the output in $OUT.
run() {
local desc=$1 want=$2; shift 2
OUT=$("$@" 2>&1); local rc=$?
if [ "$rc" = "$want" ]; then ok "$desc (rc=$rc)"; else bad "$desc: rc=$rc want $want"; echo "$OUT" | sed 's/^/ /'; fi
}
contains() { if echo "$OUT" | grep -q -- "$1"; then ok "output has '$1'"; else bad "output lacks '$1'"; echo "$OUT" | sed 's/^/ /'; fi; }
not_contains() { if echo "$OUT" | grep -q -- "$1"; then bad "output has '$1'"; echo "$OUT" | sed 's/^/ /'; else ok "output lacks '$1'"; fi; }
R() { docker exec ${P}-router "$@"; }
H() { docker exec ${P}-router /w/hc "$@"; }
# connect FROM HOST PORT: what the server says, within 3 seconds.
connect() { docker exec ${P}-$1 nc -w 3 "$2" "$3" </dev/null; }
cat > "$WORK/housecarl.yaml" <<'EOF'
listen: ["0.0.0.0:8070"]
database: /w/data/housecarl.db
ruleset: /w/data/ruleset.nft
backup: {dir: /w/backups}
log: {level: debug}
stats: {database: /w/data/stats.db}
jarl: {server: "http://127.0.0.1:8053", token: e2e-jarl-token, refresh: 10s}
EOF
{ cat "$WORK/housecarl.yaml"; echo "shadow: {enabled: true}"; } > "$WORK/housecarl-shadow.yaml"
CONF=housecarl.yaml
start() { R sh -c "(HOUSECARL_STATS_INTERVAL=3s /w/housecarld -c /w/$CONF serve >> /w/srv.log 2>&1 &); for i in \$(seq 1 30); do [ -S /run/housecarl/housecarl.sock ] && exit 0; sleep 0.3; done; exit 1"; }
stop() { docker exec ${P}-router sh -c 'kill $(pidof housecarld)'; sleep 1; }
# ---- today's router: the hand-written ruleset
sed -e 's/217\.155\.8\.194/198.18.1.2/g' -e 's/10\.0\.10\./10.88.10./g' -e 's/10\.50\.50\./10.88.50./g' \
-e 's/2a02:8013:7f3d/2001:db8:7f3d/g' "$ROOT/internal/importer/testdata/router.nft" > "$WORK/router.nft"
run "today's ruleset loads" 0 R nft -f /w/router.nft
R nft add table ip other
run "the port forward works under it" 0 connect net $WAN4 8096
contains "jellyfin"
# ---- import in shadow mode: beside today's ruleset, changing nothing
run "init" 0 R sh -c 'echo secret-pass-123 | /w/housecarld -c /w/housecarl.yaml init --password-stdin'
CONF=housecarl-shadow.yaml
run "start housecarld in shadow mode" 0 start
run "status" 0 H status
contains "Mode:.*SHADOW"
contains "Other tables:.*inet main"
run "two tables: say which" 6 H import --defines /w/router.nft -n
contains "several tables"
run "dry run of the import" 0 H import --table "inet main" --defines /w/router.nft -n
contains "5 port forwards, 2 source NAT rules"
contains "dry run: nothing was changed"
contains "dhcp (enp2s0: the define dhcp_int)"
run "zone names given at import" 0 H import --table "inet main" --defines /w/router.nft --zone enp2s0=lan --zone wg0=vpn -n
contains "lan (enp2s0: as given)"
contains "vpn (wg0: as given)"
run "import" 0 H import --table "inet main" --defines /w/router.nft
contains "applied and saved"
run "the shadow ruleset is in force" 0 R nft list table inet housecarl
contains "would dnat\] jellyfin"
contains "priority filter + 10; policy accept;"
not_contains "dnat ip to"
not_contains "maxseg size set"
not_contains "flowtable"
run "nothing in it drops, rejects or translates" 1 R sh -c "nft list table inet housecarl | grep -v 'log prefix' | sed 's/comment.*//; s/ct status dnat//' | grep -wE 'drop|reject|dnat|snat|masquerade|redirect'"
run "no warning about today's ruleset beside it" 0 H status
not_contains "takeover_pending"
not_contains "shadow_unprotected"
run "what it loads once shadow mode is off" 0 H ruleset --enforcing
contains "dnat ip to 10.88.10.57:443"
run "the port forward works, by today's ruleset" 0 connect net $WAN4 8096
contains "jellyfin"
run "and Housecarl's would-be DNAT counted it" 0 R nft list chain inet housecarl dstnat
contains 'packets [1-9][0-9]* bytes [0-9]* comment "hc:f[0-9]* \[would dnat\] jellyfin'
# A drop Housecarl has and today's ruleset doesn't: logged, let through.
SR=$(H -o id rule add forward --from $CLIENT4 --service tcp/9000 -a drop --log --first --comment "e2e shadow")
: > "$WORK/net.log"
run "a shadow drop lets the connection through" 0 docker exec ${P}-client sh -c "echo hi | nc -N -w 3 $NET4 9000"
sleep 1
run "it reached the internet" 0 grep -q "$WAN4" "$WORK/net.log"
run "the log says it would have been dropped" 0 H log
contains "would drop, let through"
run "the shadow rule goes" 0 H rule delete -y "$SR"
run "no takeover in shadow mode" 5 H takeover "inet main" -y
contains "shadow mode"
run "healthz" 0 R curl -sf http://127.0.0.1:8070/healthz
contains '"mode":"shadow"'
run "the boot ruleset is the shadow one" 0 R grep -q "would dnat" /w/data/ruleset.nft
run "the boot fallback filters nothing" 1 R grep -wE "drop|masquerade" /w/data/ruleset-fallback.nft
# Today's ruleset starts with flush ruleset: reloading it deletes
# Housecarl's table, which housecarld puts back.
run "today's ruleset reloaded" 0 R nft -f /w/router.nft
R nft add table ip other
sleep 4
run "Housecarl's table is back" 0 R nft list table inet housecarl
contains "would dnat"
run "and logged" 0 R grep -q "put it back" /w/srv.log
# ---- enforcing, and take over
stop
CONF=housecarl.yaml
run "start housecarld enforcing" 0 start
run "status" 0 H status
not_contains "SHADOW"
run "the ruleset is in force" 0 R nft list table inet housecarl
contains "dnat ip to 10.88.10.57:443"
not_contains "would"
run "zones" 0 H zone list
contains "wan.*wan.*yes"
contains "dhcp.*enp2s0"
run "port forwards" 0 H forward list
contains "jellyfin.*198.18.1.2 tcp/8096.*10.88.10.57:443"
run "take over the old table" 0 H takeover "inet main" -y
run "only Housecarl's table and the other one remain" 0 R nft list tables
not_contains "inet main"
contains "ip other"
# ---- traffic
run "port forward from the internet" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "hairpin: the public address from the LAN" 0 connect client $WAN4 8096
contains "^jellyfin$"
# The IPv6 rule stays a forward rule: jellyfin and haproxy both use inside
# port 443, so the importer doesn't guess whose pinhole it is.
run "the IPv6 rule" 0 H rule list forward
contains "to 2001:db8:7f3d:10::57 tcp/443 *accept"
run "IPv6 to the host's port 443 from the internet" 0 connect net $JF6 443
contains "jellyfin6"
run "IPv6 to another port of the host is dropped" 1 connect net $JF6 22
run "masquerade: the LAN reaches the internet as the router" 0 docker exec ${P}-client sh -c "echo hi | nc -N -w 3 $NET4 9000"
sleep 1
run "the internet saw the router's address" 0 cat "$WORK/net.log"
contains "$WAN4"
not_contains "$CLIENT4"
run "nothing else gets in from the internet" 1 connect net $JF4 443
# Antispoofing: a source that can't come from the internet.
docker exec ${P}-net ip addr add 10.9.9.9/32 dev eth0
docker exec ${P}-net ping -c 2 -W 1 -I 10.9.9.9 $WAN4 >/dev/null 2>&1
run "counters" 0 H counters
contains "b-antispoof *[1-9]"
# ---- the API from the LAN, on probation
# Allow the API from the trusted networks (through the local socket: no
# probation).
APIRULE=$(H -o id rule add input --from trusted_ips --service tcp/8070 --comment "Housecarl API")
run "the API rule was added" 0 test -n "$APIRULE"
run "probation of 10 seconds" 0 H settings set --probation 10s
TOKEN=$(R /w/housecarld -c /w/housecarl.yaml user token admin e2e 2>/dev/null | tail -1)
C() { docker exec ${P}-client /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" "$@"; }
run "the API from the LAN" 0 C whoami
run "a change over the network is on probation" 0 C --no-confirm zone add guests --interface guest0
contains "applied on probation"
run "confirmed" 0 C confirm
contains "confirmed 1 change;"
run "a change that cuts the client off" 0 C --no-confirm rule disable "$APIRULE"
contains "on probation"
run "the client is cut off (a timeout)" 124 docker exec ${P}-client timeout 3 /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" whoami
sleep 12
run "rolled back by itself: the client is back" 0 C whoami
run "the rule is enabled again" 0 H rule show "$APIRULE"
not_contains "disabled"
run "the rollback is in the history" 0 H history list
contains "rollback"
# Confirmed by the client itself: it reached the API over a new connection.
run "a change confirmed by its client" 0 C zone add guests2 --interface guest2
contains "applied and confirmed"
run "nothing waits" 0 C status
not_contains "Probation:"
run "the confirmation is audited" 0 H audit --action config.confirm
contains "confirmed automatically"
# A change that cuts its client off can't be confirmed by it: rolled back.
run "a lockout change, not confirmed" 0 C rule disable "$APIRULE"
contains "couldn't confirm it"
sleep 3
run "rolled back: the client is back" 0 C whoami
run "the rule is enabled again" 0 H rule show "$APIRULE"
not_contains "disabled"
run "a person must confirm, by setting" 0 H settings set --auto-confirm off
run "then a change waits" 0 C zone add guests3 --interface guest3
contains "applied on probation"
run "confirmed by a person" 0 C confirm
run "back to automatic" 0 H settings set --auto-confirm on
# ---- changesets: staged, reviewed, applied together
CS=$(C changeset create "e2e batch" -q)
run "a changeset" 0 test -n "$CS"
CE() { docker exec -e HOUSECARL_CHANGESET=$CS ${P}-client /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" "$@"; }
run "staged from HOUSECARL_CHANGESET" 0 CE set add e2e_block 198.51.100.7
contains "staged in changeset $CS as operation 1"
contains "from HOUSECARL_CHANGESET"
run "a rule naming the staged set" 0 CE rule add input --from e2e_block --service ssh -a drop
contains "operation 2"
run "disabling the jellyfin forward, staged" 0 CE forward update jellyfin --disabled
contains "operation 3"
run "nothing is live yet: the forward still works" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "the set isn't there yet" 4 H set show e2e_block
run "the changeset's diff" 0 CE changeset diff
contains "^+.*ip saddr @e2e_block_v4 tcp dport 22 counter drop"
contains "^-.*dnat ip to 10.88.10.57:443"
contains "valid: nftables accepts"
run "remove the forward operation" 0 CE changeset remove 3
contains "has 2 operations"
run "the diff without it" 0 CE changeset diff
not_contains "dnat ip to 10.88.10.57"
run "applied together, and confirmed by this client" 0 CE changeset apply
contains "applied and confirmed"
contains "HOUSECARL_CHANGESET still names changeset $CS"
run "one history entry" 0 H history list
contains "e2e batch"
run "confirmed" 0 C confirm
run "the set is there now" 0 H set show e2e_block
run "the forward still works" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "a stale HOUSECARL_CHANGESET is refused" 1 CE zone add stale -i stale0
contains "is applied"
CS2=$(H changeset create -q)
UNDO=$(H zone add undome -i undo0 | sed -n 's/.*history \([0-9]*\);.*/\1/p')
run "an undo is staged" 0 H -C "$CS2" history undo "$UNDO"
contains "staged in changeset $CS2"
run "not applied yet: the zone is there" 0 H zone show undome
run "an import can't be staged" 6 sh -c "echo '{\"nftables\":[]}' | docker exec -i ${P}-router /w/hc -C $CS2 import -"
contains "can't be staged"
run "discarded" 0 H changeset discard "$CS2"
run "the zone stays" 0 H zone show undome
run "a dry run shows the ruleset's diff" 0 H -n set add e2e_dry 10.1.2.3
contains "^+.*set e2e_dry_v4"
# ---- quotas: an allowance per device, kept across a restart, reset
QR=$(H -o id rule add forward --from $CLIENT4 --service tcp/9000 --quota 1mbytes,over,per-source -a drop --first --comment "e2e quota")
run "a per-device quota" 0 test -n "$QR"
send() { docker exec ${P}-client sh -c "head -c $1 /dev/zero | nc -N -w 3 $NET4 9000"; }
run "within the quota" 0 send 300000
send 2000000 >/dev/null 2>&1
run "used up: the device is cut off" 1 send 1000
run "the usage" 0 H quotas
contains "e2e quota.*1.0 MB each.*by hand"
contains "$CLIENT4 (e2e-client) *1\.[0-9] MB (1[0-9][0-9]%)"
stop; start
run "after a restart, still used up" 1 send 1000
run "the usage after a restart" 0 H quotas
contains "$CLIENT4 (e2e-client) *1\.[0-9] MB"
# A top-up: some given back (more than the cut-off packets added), then
# used up again; one address started again; then the whole quota.
run "a top-up for the device" 0 H rule reset-quota "$QR" --give 10mbytes --address $CLIENT4
contains "gave back 10.0 MB"
run "topped up: the device is back" 0 send 1000
send 30000000 >/dev/null 2>&1
run "used up again" 1 send 1000
run "an address that used nothing" 4 H rule reset-quota "$QR" --address 10.88.10.99
run "one address started again" 0 H rule reset-quota "$QR" --address $CLIENT4
run "back again" 0 send 1000
run "reset by hand" 0 H rule reset-quota "$QR"
run "the device is back" 0 send 1000
run "the reset is audited" 0 H audit --action quota.
contains "quota.reset"
contains "quota.give"
# Counted in total now: what the device used carries over; per
# destination can't take it, and says so.
run "within the quota" 0 send 300000
run "to one total" 0 H rule update "$QR" --quota 1mbytes,over
not_contains "quota_restarts"
run "the total carried over" 0 H quotas
contains "e2e quota) *[0-9.]* KB ([0-9]*%)"
run "per destination starts again" 0 H rule update "$QR" --quota 1mbytes,over,per-destination
contains "quota_restarts"
run "the quota rule goes" 0 H rule delete -y "$QR"
# ---- the zone matrix, of the imported ruleset
run "the matrix" 0 H matrix
contains "^firewall *allow"
contains "^dhcp *partly +[0-9]"
contains "dhcp → wan *.*from trusted_ips"
run "LAN to the internet" 0 H matrix dhcp wan
contains "new connections: partly allowed"
contains "allowed *from trusted_ips *rule forward #[0-9]* (Forward from trusted IPs) *[0-9]"
contains "blocked *the rest *forward policy *[0-9]"
not_contains "wg0_net"
run "the internet to the firewall" 0 H matrix wan firewall
contains "blocked *sources that can't come from outside"
contains "allowed *udp/51820"
contains "allowed *raw: icmpv6 type"
# ---- statistics (sampled every 3 seconds here)
run "traffic per device on" 0 H settings set --stats-per-device on --log-drops on
run "the device counters are in the ruleset" 0 R nft list set inet housecarl stats_up_v4
send 500000 >/dev/null 2>&1
run "a connection from the internet, dropped and logged" 1 connect net $WAN4 23
sleep 8
run "statistics are collected" 0 H stats status
contains "Collecting: *yes"
contains "Per device: *on"
run "the summary" 0 H stats --since 1h
contains "^wan *[0-9.]* [KM]B"
contains "Dropped: *[1-9]"
run "traffic per zone over time" 0 H stats traffic --since 1h --step 1m
contains "wan ↓"
run "the devices" 0 H stats top devices --since 1h
contains "^$CLIENT4 (e2e-client) *[0-9.]* KB *[0-9.]* KB"
run "each device once" 0 sh -c "docker exec ${P}-router /w/hc stats top devices --since 1h | grep -c '^$CLIENT4 ' | grep -qx 1"
run "the sources blocked" 0 H stats top blocked --since 1h
contains "^$NET4 *[1-9]"
run "what they tried" 0 H stats top tried --since 1h
contains "^tcp/23"
run "hits per rule" 0 H stats rules --since 1h
contains "input policy *[1-9]"
contains "rule #[0-9]* (deleted) *[1-9]"
run "connections over time" 0 H stats connections --since 1h --step 1m
contains "^[0-9-]* [0-9:]* *[0-9]* *[0-9]*"
# ---- connections and tracing
# A connection that stays open, masqueraded on its way out.
docker exec -d ${P}-client sh -c "sleep 30 | nc $NET4 9000"
sleep 1
run "the connection is tracked" 0 H connections --search "$CLIENT4 :9000"
contains "$CLIENT4:[0-9]* (e2e-client) *$NET4:9000 *as $WAN4"
run "without names" 0 H connections --search "$CLIENT4 :9000" --no-names
contains "$CLIENT4:[0-9]* *$NET4:9000 *as $WAN4"
run "closed" 0 H connections close --search "$CLIENT4 :9000" -y
contains "closed [1-9][0-9]* connection"
run "gone" 0 H connections --search "$CLIENT4 :9000"
not_contains ":9000"
# A block added while traffic flows: it stops new connections, not this
# one, unless it checks every packet.
docker exec -d ${P}-client sh -c "rm -f /tmp/ncdone; yes | nc $NET4 9000; touch /tmp/ncdone"
sleep 1
BR=$(H -o id rule add forward --from $CLIENT4 --service tcp/9000 -a reject --reject-with tcp-reset --comment "e2e block" --first)
run "a block for new connections" 0 test -n "$BR"
sleep 2
run "the open upload carries on" 1 docker exec ${P}-client test -f /tmp/ncdone
run "the block checks every packet now" 0 H rule update "$BR" --every-packet
sleep 2
run "the upload is cut" 0 docker exec ${P}-client test -f /tmp/ncdone
# Tracing a new connection: dropped, by that rule.
docker exec -d ${P}-router sh -c "/w/hc trace --address $CLIENT4 --port 9000 --seconds 6 > /w/trace.out 2>&1"
sleep 2
connect client $NET4 9000 >/dev/null 2>&1
sleep 6
run "the trace" 0 cat "$WORK/trace.out"
contains "$CLIENT4:[0-9]* -> $NET4:9000 tcp syn"
contains "inet housecarl forward: rule forward #$BR (e2e block) -> drop"
contains "=> DROPPED by rule forward #$BR (e2e block)"
run "no trace table is left" 0 R nft list tables
not_contains "housecarl-trace"
run "the block goes" 0 H rule delete -y "$BR"
# ---- the drop log
run "log drops from the internet to the API" 0 H rule add input --in wan --service tcp/8070 -a drop --log --first
connect net $WAN4 8070 >/dev/null 2>&1
sleep 1
run "the log shows it" 0 H log
contains "hc input #.* drop.*198.18.1.10:.*198.18.1.2:8070"
# ---- the flowtable, and devices that appear later
run "a flowtable device that doesn't exist yet" 0 H settings set --flowtable wan --flowtable enp2s0 --flowtable hcft0
contains "flowtable_device_missing"
run "the flowtable has the devices that exist" 0 R nft list flowtables
contains "devices = { enp2s0, wan }"
run "forwarded through the flowtable" 0 connect net $WAN4 8096
contains "jellyfin"
run "the device appears" 0 R ip link add hcft0 type veth peer name hcft1
R ip link set hcft0 up
sleep 4
run "applied again with it" 0 R nft list flowtables
contains "hcft0"
run "logged" 0 R grep -c "interfaces changed: applied the ruleset again" /w/srv.log
run "it comes back as a new interface" 0 R sh -c 'ip link del hcft0 && ip link add hcft0 type veth peer name hcft1 && ip link set hcft0 up'
sleep 4
run "applied again" 0 R sh -c 'grep -c "interfaces changed: applied the ruleset again" /w/srv.log | grep -qx 2'
run "the flowtable still has it" 0 R nft list flowtables
contains "hcft0"
# ---- VLAN sub-interfaces: tagged traffic between the router and the client
R sh -c 'ip link add link enp2s0 name enp2s0.30 type vlan id 30 && ip addr add 10.88.30.1/24 dev enp2s0.30 && ip link set enp2s0.30 up'
docker exec ${P}-client sh -c 'ip link add link eth0 name eth0.30 type vlan id 30 && ip addr add 10.88.30.20/24 dev eth0.30 && ip link set eth0.30 up'
run "a zone of a VLAN" 0 H zone add iot --interface enp2s0.30
run "nothing from the VLAN to the API yet" 1 connect client 10.88.30.1 8070
VR=$(H -o id rule add input --in iot --service tcp/8070 --comment "e2e vlan")
run "a rule for the VLAN's zone" 0 test -n "$VR"
run "the VLAN reaches the API" 0 connect client 10.88.30.1 8070
run "the rule counted it" 0 H rule list input
contains "e2e vlan"
run "the flowtable on the VLAN" 0 H settings set --flowtable wan --flowtable enp2s0 --flowtable hcft0 --flowtable enp2s0.30
run "attached" 0 R nft list flowtables
contains "enp2s0.30"
run "the VLAN rule goes" 0 H rule delete -y "$VR"
run "the VLAN zone goes" 0 H zone delete -y iot
R ip link del enp2s0.30
# ---- ban lists: bans kept from the firewall, a host, or everything
# DNS is open to the internet on the router (tcp/53): a listener answers.
docker exec -d ${P}-router sh -c 'while [ ! -f /tmp/nodns ]; do echo dns | nc -N -l 0.0.0.0 53; done'
sleep 1
run "the router's tcp/53 from the internet" 0 connect net $WAN4 53
contains "^dns$"
run "a ban list for the firewall" 0 H banlist add router --scope firewall
# The lab's internet host shares the router's WAN /64: ban IPv6 addresses
# alone (a /64 would hold the router's own, and is refused).
run "a ban list for jellyfin" 0 H banlist add jf --scope hosts --host $JF4 --host $JF6 --group-v6 128
run "a ban list for everything" 0 H banlist add edge --scope everything --max-time 1h
run "the lists" 0 H banlist list
contains "jf *hosts *10.88.10.57, 2001:db8:7f3d:10::57 *0"
# The firewall's own DNS, for the internet host.
run "ban from the firewall's tcp/53" 0 H ban router $NET4 --tcp dns --reason "e2e"
contains "banned $NET4, tcp/53, in router"
run "the router's DNS is cut for it" 1 connect net $WAN4 53
run "the port forward isn't (scope firewall)" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "the ban in the ruleset" 0 R nft list ruleset
contains "ip saddr . meta l4proto . th dport @ban_[0-9]*_v4_svc counter packets [1-9]"
run "lift it" 0 H unban router $NET4
contains "1 bans lifted"
run "DNS again" 0 connect net $WAN4 53
contains "^dns$"
# fail2ban on jellyfin, with Housecarl's action and a token for its list.
run "a user for jellyfin's fail2ban" 0 H user add f2b-jf --no-password --grant bans:write:jf
R sh -c "/w/hc token create jf --user f2b-jf > /w/f2b.token 2>/dev/null"
run "its token" 0 test -s "$WORK/f2b.token"
# f2b ACTION ip=… port=…: the action's command from contrib/, with the
# jail's values, as fail2ban runs it (on jellyfin).
f2b() {
local conf="$ROOT/contrib/fail2ban/action.d/housecarl.conf" a=$1; shift
local cmd curl
cmd=$(awk -v a="$a" '$0 ~ "^"a" *=" {sub(/^[^=]*= */,""); print; on=1; next} on && /^[ \t]/ {sub(/^[ \t]+/,""); print; next} {on=0}' "$conf" | tr '\n' ' ')
curl=$(sed -n 's/^_curl *= *//p' "$conf")
cmd=${cmd//<_curl>/$curl}
local kv
for kv in "$@" "list=jf" "url=http://$LAN4:8070" "tokenfile=/w/f2b.token" "curlopts=" "protocol=tcp" "bantime=600" "name=jellyfin"; do
cmd=${cmd//<${kv%%=*}>/${kv#*=}}
done
docker exec ${P}-jf sh -c "$cmd"
}
run "fail2ban bans the internet host's https" 0 f2b actionban ip=$NET4 port=https
run "the port forward to jellyfin is cut for it" 1 connect net $WAN4 8096
run "the router's DNS isn't (scope hosts)" 0 connect net $WAN4 53
run "fail2ban bans its IPv6 address" 0 f2b actionban ip=$NET6 port=https
run "IPv6 to jellyfin's 443 is cut" 1 connect net $JF6 443
run "the bans" 0 H bans jf
contains "$NET4 *tcp/443 .*f2b-jf *fail2ban/jellyfin"
contains "$NET6 *tcp/443"
run "never the inside" 22 f2b actionban ip=$CLIENT4 port=https
run "refused, audited" 0 H audit --action ban.add
contains "failure.*inside"
run "fail2ban unbans" 0 f2b actionunban ip=$NET4
run "the port forward works again" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "unbanning what Housecarl already let go is no error" 0 f2b actionunban ip=$NET4
run "and the IPv6 ban" 0 f2b actionunban ip=$NET6
run "jellyfin by IPv6 again" 0 connect net $JF6 443
contains "jellyfin6"
run "the token can't ban elsewhere" 22 f2b actionban ip=$NET4 port=https list=router
# Everything, cutting a connection that is open, and kept over a restart.
R touch /tmp/nodns
connect net $WAN4 53 >/dev/null 2>&1
docker exec -d ${P}-router sh -c 'nc -l 0.0.0.0 53 > /w/held.out'
sleep 1
docker exec -d ${P}-net sh -c "sleep 30 | nc $WAN4 53"
sleep 1
run "an open connection from the internet host" 0 H connections --search "$NET4 :53"
contains "$NET4:[0-9]* *$WAN4:53"
run "ban it from everything" 0 H ban edge $NET4 --time 1d
contains "banned $NET4, every port, in edge (ends in 1h)"
contains "[1-9][0-9]* connections closed"
run "its connection is gone" 0 H connections --search "$NET4 :53"
not_contains ":53"
run "nothing gets through for it" 1 connect net $WAN4 8096
stop
run "restart housecarld" 0 start
run "the ban is back after the restart" 0 R nft list ruleset
contains "$NET4 timeout [0-9hms]* expires"
run "still nothing for it" 1 connect net $WAN4 8096
run "lift it" 0 H unban edge $NET4
run "the ban is out of the ruleset" 0 R nft list ruleset
not_contains "$NET4 timeout"
run "the port forward again" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "the lists go" 0 sh -c "docker exec ${P}-router /w/hc banlist delete -y router && docker exec ${P}-router /w/hc banlist delete -y jf && docker exec ${P}-router /w/hc banlist delete -y edge"
# ---- renaming a zone others refer to
run "rename the imported dhcp zone" 0 H zone update dhcp --name lan
run "its rules follow" 0 H rule list input
contains "lan"
not_contains "dhcp "
run "the ruleset is unchanged by it" 0 R nft list chain inet housecarl input
contains 'iifname "enp2s0" udp dport 67'
# ---- a set from Jarl: names' addresses, kept up to date
# A fake Jarl on the router: example.com in views internal and external.
# jf's AAAA is jellyfin's; ssh.example.com is a CNAME for it; the internet
# host's address is outside the LAN.
jarl_data() { # JF_AAAA
cat > "$WORK/jarl.json" <<EOJ
{"zones": [{"id": 1, "name": "example.com.", "view": "internal", "type": "primary"},
{"id": 2, "name": "example.com.", "view": "external", "type": "primary"}],
"records": {"1": [{"name": "jf.example.com.", "type": "A", "data": "$JF4"},
{"name": "jf.example.com.", "type": "AAAA", "data": "$1"},
{"name": "ssh.example.com.", "type": "CNAME", "data": "jf.example.com."},
{"name": "net.example.com.", "type": "AAAA", "data": "$NET6"}],
"2": [{"name": "ssh.example.com.", "type": "AAAA", "data": "$NET6"}]}}
EOJ
}
jarl_data $JF6
docker exec -d ${P}-router sh -c '/w/fakejarl -listen 127.0.0.1:8053 -data /w/jarl.json -token e2e-jarl-token >> /w/fakejarl.log 2>&1'
sleep 1
run "IPv6 ssh to jellyfin is dropped" 1 connect net $JF6 22
run "a set from Jarl" 0 H set add jfssh --jarl ssh.example.com --within lan
run "a name in two views needs its view" 0 H set jarl jfssh --refresh
contains "in Jarl's views external, internal: name the view"
run "name the views" 0 H set update jfssh --jarl ssh.example.com@internal --jarl net.example.com@internal
JR=$(H -o id rule add forward --in wan --to jfssh --service ssh --first --comment "e2e jarl")
run "the set's addresses, CNAME followed" 0 H set jarl jfssh --refresh
contains "ssh.example.com → jf.example.com *internal *$JF4 *yes"
contains "ssh.example.com → jf.example.com *internal *$JF6 *yes"
contains "net.example.com *internal *$NET6 *no: not within lan"
run "in the ruleset" 0 R nft list set inet housecarl jfssh_v6
contains "$JF6"
not_contains "$NET6"
run "the address outside is warned of" 0 H status
contains "jarl_outside"
run "IPv6 ssh to jellyfin through the Jarl set" 0 connect net $JF6 22
contains "^ssh$"
# The record changes in Jarl: within the refresh, the set follows.
jarl_data 2001:db8:7f3d:10::99
sleep 12
run "the new address, without a change in Housecarl" 0 R nft list set inet housecarl jfssh_v6
contains "2001:db8:7f3d:10::99"
not_contains "$JF6"
run "ssh to the old address is dropped again" 1 connect net $JF6 22
run "no history entry for it" 0 H history list
contains "update set jfssh"
not_contains "10::99"
run "an audit record" 0 H audit --action set.jarl
contains "added 2001:db8:7f3d:10::99"
# Jarl goes away: the set keeps what it had, over a restart too.
jarl_data $JF6
H set jarl jfssh --refresh >/dev/null
R sh -c 'kill $(pidof fakejarl)'
run "Jarl down: the set keeps its addresses" 0 H set jarl jfssh --refresh
contains "Reading Jarl failed"
contains "$JF6 *yes"
run "still through" 0 connect net $JF6 22
contains "^ssh$"
run "warned" 0 H status
contains "jarl_unreachable"
stop
run "restart housecarld, Jarl still down" 0 start
run "the set is back from the database" 0 R nft list set inet housecarl jfssh_v6
contains "$JF6"
run "the boot ruleset has its addresses" 0 R grep -q "$JF6" /w/data/ruleset.nft
run "still through after the restart" 0 connect net $JF6 22
contains "^ssh$"
run "the Jarl rule goes" 0 H rule delete -y "$JR"
run "the Jarl set goes" 0 H set delete -y jfssh
run "no Jarl warning left" 0 H status
not_contains "jarl_"
# ---- restart and boot
stop
run "the ruleset stays without housecarld" 0 connect net $WAN4 8096
contains "jellyfin"
run "restart housecarld" 0 start
run "after the restart" 0 connect client $WAN4 8096
contains "jellyfin"
run "the boot ruleset" 0 R test -s /w/data/ruleset.nft
R nft flush ruleset
run "nothing forwarded after a flush" 1 connect net $WAN4 8096
run "the boot unit's command restores it" 0 R nft -f /w/data/ruleset.nft
run "forwarded again" 0 connect net $WAN4 8096
contains "jellyfin"
run "the boot ruleset has no flowtable (its devices may not exist yet)" 1 R grep -q flowtable /w/data/ruleset.nft
run "healthz" 0 R curl -sf http://127.0.0.1:8070/healthz
contains '"firewall":"applied"'
# ---- the boot fallback: what the boot unit runs, with the lab's paths
UNIT=$(sed -n 's/^ExecStart=//p' "$ROOT/deploy/systemd/housecarl-ruleset.service" | sed 's|/var/lib/housecarl|/w/data|g; s|/usr/sbin/nft|nft|g')
run "the fallback was saved" 0 R test -s /w/data/ruleset-fallback.nft
R cp /w/data/ruleset.nft /w/ruleset.good
R sh -c 'echo "this is not nftables" >> /w/data/ruleset.nft'
R nft flush ruleset
run "the boot unit fails, and loads the fallback" 1 R sh -c "$UNIT"
contains "the saved ruleset failed to load"
run "the fallback is in force" 0 R nft list table inet housecarl
contains "hc:fallback-input"
run "nothing new gets in from the internet" 1 connect net $WAN4 8096
run "not to the firewall either" 1 connect net $WAN4 8070
: > "$WORK/net.log"
run "the LAN still reaches the internet" 0 connect client $NET4 9000
sleep 1
run "masqueraded" 0 grep -q "$WAN4" "$WORK/net.log"
run "healthz says so" 0 R sh -c 'curl -s -w " %{http_code}" http://127.0.0.1:8070/healthz'
contains '"firewall":"fallback"'
contains " 503"
run "status says so" 0 H status
contains "NOT in force: the boot fallback"
stop
run "housecarld starts" 0 start
run "and puts the real ruleset back" 0 connect net $WAN4 8096
contains "jellyfin"
run "and saves a good boot ruleset" 0 R nft -c -f /w/data/ruleset.nft
run "healthz is happy again" 0 R curl -sf http://127.0.0.1:8070/healthz
contains '"firewall":"applied"'
# ---- backup
run "backup" 0 R /w/housecarld -c /w/housecarl.yaml backup
contains "/w/backups/housecarl-"
echo
echo "RESULT: $PASS passed, $FAIL failed"
[ $FAIL = 0 ]
Reference in new issue View Git Blame Copy Permalink