adam/housecarl
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity
Files
540f4b85ac639682dcf33408f18f3d1af38e0984
housecarl/scripts/e2e.sh
T
abrown38andClaude Opus 5.5 500bfc7bdd Ban lists: where Guard and fail2ban ban addresses 
A program bans an address (every port, or a protocol's ports) for a
time; the ban list says what it is kept from: the firewall, some hosts
behind it, or everything. Bans drop ahead of every rule, close the
address's connections, group IPv6 by /64, are kept in the database
across restarts and audited. Never banned: the inside, the firewall's
own addresses, the caller, the list's never. bans grant (bans:write:LIST),
hc banlist/bans/ban/unban, Traffic -> Bans, a fail2ban action in contrib/,
and failed logins and refused tokens logged for Guard and fail2ban.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 16:29:37 +01:00

638 lines
31 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# End-to-end test with real traffic: a Debian 13 router container (nftables
# 1.1.3) between an "internet" network and a LAN, IPv4 and IPv6, runs
# housecarld. The maintainer's own router ruleset (internal/importer/
# testdata/router.nft, with lab addresses) is loaded as it is today,
# imported with housecarld in shadow mode beside it, taken over, and then
# checked with connections: port forwards, hairpin NAT, the IPv6 pinhole,
# masquerading, antispoofing, the drop log, a change on probation that cuts
# the client off and rolls itself back, a restart and the boot ruleset.
# Needs Docker; no root on the host. Run from the repository root:
#
# make e2e or scripts/e2e.sh
#
# KEEP=1 leaves the containers and networks running. Set DOCKER_HOST if
# Docker isn't at its default socket.
set -u
ROOT=$(cd "$(dirname "$0")/.." && pwd)
P=${E2E_PREFIX:-hce2e}
IMAGE=${E2E_IMAGE:-housecarl-nft}
WORK=${WORK:-$(mktemp -d "${TMPDIR:-/tmp}/housecarl-e2e.XXXXXX")}
docker info >/dev/null 2>&1 || { echo "Docker isn't reachable (set DOCKER_HOST?)"; exit 2; }
(cd "$ROOT" && CGO_ENABLED=0 go build -o "$WORK/housecarld" ./cmd/housecarld && CGO_ENABLED=0 go build -o "$WORK/hc" ./cmd/hc) || exit 1
# The image: Debian 13 with nftables and network tools. 'docker build'
# can't reach the network on some hosts, so it is made with run + commit.
if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
docker rm -f ${P}-build >/dev/null 2>&1
docker run --name ${P}-build debian:trixie sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get update -qq && apt-get install -y -qq --no-install-recommends nftables iproute2 iputils-ping netcat-openbsd curl ca-certificates >/dev/null 2>&1 && rm -rf /var/lib/apt/lists/*' || exit 1
docker commit ${P}-build "$IMAGE" >/dev/null && docker rm ${P}-build >/dev/null || exit 1
fi
cleanup() {
if [ "${KEEP:-}" = 1 ]; then echo "left running: containers ${P}-*, networks ${P}-*, files in $WORK"; return; fi
docker exec ${P}-router sh -c 'rm -rf /w/data /w/backups /w/*.log' >/dev/null 2>&1
docker rm -f $(docker ps -aq --filter "name=^${P}-") >/dev/null 2>&1
docker network rm ${P}-wan ${P}-lan ${P}-wg >/dev/null 2>&1
rm -rf "$WORK"
}
trap cleanup EXIT
docker rm -f $(docker ps -aq --filter "name=^${P}-") >/dev/null 2>&1
docker network rm ${P}-wan ${P}-lan ${P}-wg >/dev/null 2>&1
# The lab: "the internet" 198.18.1.0/24 and 2001:db8:1::/64; the LAN
# 10.88.10.0/24 and 2001:db8:7f3d:10::/64 (the /48 is the delegated
# prefix); a WireGuard-like network 10.88.50.0/24. The router has .1 on its
# inside networks (Docker's gateway moves to .254).
docker network create --ipv6 --subnet 198.18.1.0/24 --gateway 198.18.1.254 --subnet 2001:db8:1::/64 --gateway 2001:db8:1::fe ${P}-wan >/dev/null || exit 1
docker network create --ipv6 --subnet 10.88.10.0/24 --gateway 10.88.10.254 --subnet 2001:db8:7f3d:10::/64 --gateway 2001:db8:7f3d:10::fe ${P}-lan >/dev/null || exit 1
docker network create --subnet 10.88.50.0/24 --gateway 10.88.50.254 ${P}-wg >/dev/null || exit 1
WAN4=198.18.1.2 WAN6=2001:db8:1::2 NET4=198.18.1.10 NET6=2001:db8:1::10
LAN4=10.88.10.1 LAN6=2001:db8:7f3d:10::1 JF4=10.88.10.57 JF6=2001:db8:7f3d:10::57 CLIENT4=10.88.10.20
sysctls="--sysctl net.ipv4.ip_forward=1 --sysctl net.ipv6.conf.all.forwarding=1 --sysctl net.ipv6.conf.all.keep_addr_on_down=1 --sysctl net.ipv6.conf.default.keep_addr_on_down=1"
docker run -d --name ${P}-router --hostname router --cap-add NET_ADMIN $sysctls \
--network ${P}-wan --ip $WAN4 --ip6 $WAN6 -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1
docker network connect --ip $LAN4 --ip6 $LAN6 ${P}-lan ${P}-router || exit 1
docker network connect --ip 10.88.50.1 ${P}-wg ${P}-router || exit 1
# The client has a name for housecarld's reverse lookups (/etc/hosts
# here, a PTR record in real life); the addresses outside get none.
docker exec ${P}-router sh -c "echo '$CLIENT4 e2e-client.lab' >> /etc/hosts" || exit 1
# The router's interfaces get the names of the real one: wan, enp2s0, wg0
# (attached in that order: eth0, eth1, eth2).
docker exec ${P}-router sh -c 'for p in eth0:wan eth1:enp2s0 eth2:wg0; do i=${p%:*} n=${p#*:}; ip link set $i down && ip link set $i name $n && ip link set $n up; done
ip route replace default via 198.18.1.254' || exit 1
# Hosts: one on the internet, a server and a client on the LAN; inside
# hosts route through the router, the internet reaches the delegated
# prefix through it.
host() { # NAME NETWORK IP4 IP6 GATEWAY4 GATEWAY6
docker run -d --name ${P}-$1 --cap-add NET_ADMIN --sysctl net.ipv6.conf.all.keep_addr_on_down=1 --network ${P}-$2 --ip $3 ${4:+--ip6 $4} -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1
docker exec ${P}-$1 sh -c "ip route replace default via $5 ${6:+&& ip -6 route replace default via $6}" || exit 1
}
host net wan $NET4 $NET6 198.18.1.254
docker exec ${P}-net ip -6 route add 2001:db8:7f3d::/48 via $WAN6
host jf lan $JF4 $JF6 $LAN4 $LAN6
host client lan $CLIENT4 2001:db8:7f3d:10::20 $LAN4 $LAN6
# Servers: jellyfin on 443 (IPv4 and IPv6) and 22, a listener on the
# internet that logs who connects.
docker exec -d ${P}-jf sh -c 'while true; do echo jellyfin | nc -N -l 0.0.0.0 443; done'
docker exec -d ${P}-jf sh -c 'while true; do echo jellyfin6 | nc -6 -N -l :: 443; done'
docker exec -d ${P}-jf sh -c 'while true; do echo ssh | nc -6 -N -l :: 22; done'
docker exec -d ${P}-net sh -c 'nc -lkvn 0.0.0.0 9000 2>>/w/net.log >/dev/null'
PASS=0; FAIL=0
ok() { echo "PASS: $1"; PASS=$((PASS+1)); }
bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); }
# run DESC EXPECTED_RC CMD...: runs a command, checks its exit code and
# keeps the output in $OUT.
run() {
local desc=$1 want=$2; shift 2
OUT=$("$@" 2>&1); local rc=$?
if [ "$rc" = "$want" ]; then ok "$desc (rc=$rc)"; else bad "$desc: rc=$rc want $want"; echo "$OUT" | sed 's/^/ /'; fi
}
contains() { if echo "$OUT" | grep -q -- "$1"; then ok "output has '$1'"; else bad "output lacks '$1'"; echo "$OUT" | sed 's/^/ /'; fi; }
not_contains() { if echo "$OUT" | grep -q -- "$1"; then bad "output has '$1'"; echo "$OUT" | sed 's/^/ /'; else ok "output lacks '$1'"; fi; }
R() { docker exec ${P}-router "$@"; }
H() { docker exec ${P}-router /w/hc "$@"; }
# connect FROM HOST PORT: what the server says, within 3 seconds.
connect() { docker exec ${P}-$1 nc -w 3 "$2" "$3" </dev/null; }
cat > "$WORK/housecarl.yaml" <<'EOF'
listen: ["0.0.0.0:8070"]
database: /w/data/housecarl.db
ruleset: /w/data/ruleset.nft
backup: {dir: /w/backups}
log: {level: debug}
stats: {database: /w/data/stats.db}
EOF
{ cat "$WORK/housecarl.yaml"; echo "shadow: {enabled: true}"; } > "$WORK/housecarl-shadow.yaml"
CONF=housecarl.yaml
start() { R sh -c "(HOUSECARL_STATS_INTERVAL=3s /w/housecarld -c /w/$CONF serve >> /w/srv.log 2>&1 &); for i in \$(seq 1 30); do [ -S /run/housecarl/housecarl.sock ] && exit 0; sleep 0.3; done; exit 1"; }
stop() { docker exec ${P}-router sh -c 'kill $(pidof housecarld)'; sleep 1; }
# ---- today's router: the hand-written ruleset
sed -e 's/217\.155\.8\.194/198.18.1.2/g' -e 's/10\.0\.10\./10.88.10./g' -e 's/10\.50\.50\./10.88.50./g' \
-e 's/2a02:8013:7f3d/2001:db8:7f3d/g' "$ROOT/internal/importer/testdata/router.nft" > "$WORK/router.nft"
run "today's ruleset loads" 0 R nft -f /w/router.nft
R nft add table ip other
run "the port forward works under it" 0 connect net $WAN4 8096
contains "jellyfin"
# ---- import in shadow mode: beside today's ruleset, changing nothing
run "init" 0 R sh -c 'echo secret-pass-123 | /w/housecarld -c /w/housecarl.yaml init --password-stdin'
CONF=housecarl-shadow.yaml
run "start housecarld in shadow mode" 0 start
run "status" 0 H status
contains "Mode:.*SHADOW"
contains "Other tables:.*inet main"
run "two tables: say which" 6 H import --defines /w/router.nft -n
contains "several tables"
run "dry run of the import" 0 H import --table "inet main" --defines /w/router.nft -n
contains "5 port forwards, 2 source NAT rules"
contains "dry run: nothing was changed"
contains "dhcp (enp2s0: the define dhcp_int)"
run "zone names given at import" 0 H import --table "inet main" --defines /w/router.nft --zone enp2s0=lan --zone wg0=vpn -n
contains "lan (enp2s0: as given)"
contains "vpn (wg0: as given)"
run "import" 0 H import --table "inet main" --defines /w/router.nft
contains "applied and saved"
run "the shadow ruleset is in force" 0 R nft list table inet housecarl
contains "would dnat\] jellyfin"
contains "priority filter + 10; policy accept;"
not_contains "dnat ip to"
not_contains "maxseg size set"
not_contains "flowtable"
run "nothing in it drops, rejects or translates" 1 R sh -c "nft list table inet housecarl | grep -v 'log prefix' | sed 's/comment.*//; s/ct status dnat//' | grep -wE 'drop|reject|dnat|snat|masquerade|redirect'"
run "no warning about today's ruleset beside it" 0 H status
not_contains "takeover_pending"
not_contains "shadow_unprotected"
run "what it loads once shadow mode is off" 0 H ruleset --enforcing
contains "dnat ip to 10.88.10.57:443"
run "the port forward works, by today's ruleset" 0 connect net $WAN4 8096
contains "jellyfin"
run "and Housecarl's would-be DNAT counted it" 0 R nft list chain inet housecarl dstnat
contains 'packets [1-9][0-9]* bytes [0-9]* comment "hc:f[0-9]* \[would dnat\] jellyfin'
# A drop Housecarl has and today's ruleset doesn't: logged, let through.
SR=$(H -o id rule add forward --from $CLIENT4 --service tcp/9000 -a drop --log --first --comment "e2e shadow")
: > "$WORK/net.log"
run "a shadow drop lets the connection through" 0 docker exec ${P}-client sh -c "echo hi | nc -N -w 3 $NET4 9000"
sleep 1
run "it reached the internet" 0 grep -q "$WAN4" "$WORK/net.log"
run "the log says it would have been dropped" 0 H log
contains "would drop, let through"
run "the shadow rule goes" 0 H rule delete -y "$SR"
run "no takeover in shadow mode" 5 H takeover "inet main" -y
contains "shadow mode"
run "healthz" 0 R curl -sf http://127.0.0.1:8070/healthz
contains '"mode":"shadow"'
run "the boot ruleset is the shadow one" 0 R grep -q "would dnat" /w/data/ruleset.nft
run "the boot fallback filters nothing" 1 R grep -wE "drop|masquerade" /w/data/ruleset-fallback.nft
# Today's ruleset starts with flush ruleset: reloading it deletes
# Housecarl's table, which housecarld puts back.
run "today's ruleset reloaded" 0 R nft -f /w/router.nft
R nft add table ip other
sleep 4
run "Housecarl's table is back" 0 R nft list table inet housecarl
contains "would dnat"
run "and logged" 0 R grep -q "put it back" /w/srv.log
# ---- enforcing, and take over
stop
CONF=housecarl.yaml
run "start housecarld enforcing" 0 start
run "status" 0 H status
not_contains "SHADOW"
run "the ruleset is in force" 0 R nft list table inet housecarl
contains "dnat ip to 10.88.10.57:443"
not_contains "would"
run "zones" 0 H zone list
contains "wan.*wan.*yes"
contains "dhcp.*enp2s0"
run "port forwards" 0 H forward list
contains "jellyfin.*198.18.1.2 tcp/8096.*10.88.10.57:443"
run "take over the old table" 0 H takeover "inet main" -y
run "only Housecarl's table and the other one remain" 0 R nft list tables
not_contains "inet main"
contains "ip other"
# ---- traffic
run "port forward from the internet" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "hairpin: the public address from the LAN" 0 connect client $WAN4 8096
contains "^jellyfin$"
# The IPv6 rule stays a forward rule: jellyfin and haproxy both use inside
# port 443, so the importer doesn't guess whose pinhole it is.
run "the IPv6 rule" 0 H rule list forward
contains "to 2001:db8:7f3d:10::57 tcp/443 *accept"
run "IPv6 to the host's port 443 from the internet" 0 connect net $JF6 443
contains "jellyfin6"
run "IPv6 to another port of the host is dropped" 1 connect net $JF6 22
run "masquerade: the LAN reaches the internet as the router" 0 docker exec ${P}-client sh -c "echo hi | nc -N -w 3 $NET4 9000"
sleep 1
run "the internet saw the router's address" 0 cat "$WORK/net.log"
contains "$WAN4"
not_contains "$CLIENT4"
run "nothing else gets in from the internet" 1 connect net $JF4 443
# Antispoofing: a source that can't come from the internet.
docker exec ${P}-net ip addr add 10.9.9.9/32 dev eth0
docker exec ${P}-net ping -c 2 -W 1 -I 10.9.9.9 $WAN4 >/dev/null 2>&1
run "counters" 0 H counters
contains "b-antispoof *[1-9]"
# ---- the API from the LAN, on probation
# Allow the API from the trusted networks (through the local socket: no
# probation).
APIRULE=$(H -o id rule add input --from trusted_ips --service tcp/8070 --comment "Housecarl API")
run "the API rule was added" 0 test -n "$APIRULE"
run "probation of 10 seconds" 0 H settings set --probation 10s
TOKEN=$(R /w/housecarld -c /w/housecarl.yaml user token admin e2e 2>/dev/null | tail -1)
C() { docker exec ${P}-client /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" "$@"; }
run "the API from the LAN" 0 C whoami
run "a change over the network is on probation" 0 C --no-confirm zone add guests --interface guest0
contains "applied on probation"
run "confirmed" 0 C confirm
contains "confirmed 1 change;"
run "a change that cuts the client off" 0 C --no-confirm rule disable "$APIRULE"
contains "on probation"
run "the client is cut off (a timeout)" 124 docker exec ${P}-client timeout 3 /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" whoami
sleep 12
run "rolled back by itself: the client is back" 0 C whoami
run "the rule is enabled again" 0 H rule show "$APIRULE"
not_contains "disabled"
run "the rollback is in the history" 0 H history list
contains "rollback"
# Confirmed by the client itself: it reached the API over a new connection.
run "a change confirmed by its client" 0 C zone add guests2 --interface guest2
contains "applied and confirmed"
run "nothing waits" 0 C status
not_contains "Probation:"
run "the confirmation is audited" 0 H audit --action config.confirm
contains "confirmed automatically"
# A change that cuts its client off can't be confirmed by it: rolled back.
run "a lockout change, not confirmed" 0 C rule disable "$APIRULE"
contains "couldn't confirm it"
sleep 3
run "rolled back: the client is back" 0 C whoami
run "the rule is enabled again" 0 H rule show "$APIRULE"
not_contains "disabled"
run "a person must confirm, by setting" 0 H settings set --auto-confirm off
run "then a change waits" 0 C zone add guests3 --interface guest3
contains "applied on probation"
run "confirmed by a person" 0 C confirm
run "back to automatic" 0 H settings set --auto-confirm on
# ---- changesets: staged, reviewed, applied together
CS=$(C changeset create "e2e batch" -q)
run "a changeset" 0 test -n "$CS"
CE() { docker exec -e HOUSECARL_CHANGESET=$CS ${P}-client /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" "$@"; }
run "staged from HOUSECARL_CHANGESET" 0 CE set add e2e_block 198.51.100.7
contains "staged in changeset $CS as operation 1"
contains "from HOUSECARL_CHANGESET"
run "a rule naming the staged set" 0 CE rule add input --from e2e_block --service ssh -a drop
contains "operation 2"
run "disabling the jellyfin forward, staged" 0 CE forward update jellyfin --disabled
contains "operation 3"
run "nothing is live yet: the forward still works" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "the set isn't there yet" 4 H set show e2e_block
run "the changeset's diff" 0 CE changeset diff
contains "^+.*ip saddr @e2e_block_v4 tcp dport 22 counter drop"
contains "^-.*dnat ip to 10.88.10.57:443"
contains "valid: nftables accepts"
run "remove the forward operation" 0 CE changeset remove 3
contains "has 2 operations"
run "the diff without it" 0 CE changeset diff
not_contains "dnat ip to 10.88.10.57"
run "applied together, and confirmed by this client" 0 CE changeset apply
contains "applied and confirmed"
contains "HOUSECARL_CHANGESET still names changeset $CS"
run "one history entry" 0 H history list
contains "e2e batch"
run "confirmed" 0 C confirm
run "the set is there now" 0 H set show e2e_block
run "the forward still works" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "a stale HOUSECARL_CHANGESET is refused" 1 CE zone add stale -i stale0
contains "is applied"
CS2=$(H changeset create -q)
UNDO=$(H zone add undome -i undo0 | sed -n 's/.*history \([0-9]*\);.*/\1/p')
run "an undo is staged" 0 H -C "$CS2" history undo "$UNDO"
contains "staged in changeset $CS2"
run "not applied yet: the zone is there" 0 H zone show undome
run "an import can't be staged" 6 sh -c "echo '{\"nftables\":[]}' | docker exec -i ${P}-router /w/hc -C $CS2 import -"
contains "can't be staged"
run "discarded" 0 H changeset discard "$CS2"
run "the zone stays" 0 H zone show undome
run "a dry run shows the ruleset's diff" 0 H -n set add e2e_dry 10.1.2.3
contains "^+.*set e2e_dry_v4"
# ---- quotas: an allowance per device, kept across a restart, reset
QR=$(H -o id rule add forward --from $CLIENT4 --service tcp/9000 --quota 1mbytes,over,per-source -a drop --first --comment "e2e quota")
run "a per-device quota" 0 test -n "$QR"
send() { docker exec ${P}-client sh -c "head -c $1 /dev/zero | nc -N -w 3 $NET4 9000"; }
run "within the quota" 0 send 300000
send 2000000 >/dev/null 2>&1
run "used up: the device is cut off" 1 send 1000
run "the usage" 0 H quotas
contains "e2e quota.*1.0 MB each.*by hand"
contains "$CLIENT4 (e2e-client) *1\.[0-9] MB (1[0-9][0-9]%)"
stop; start
run "after a restart, still used up" 1 send 1000
run "the usage after a restart" 0 H quotas
contains "$CLIENT4 (e2e-client) *1\.[0-9] MB"
# A top-up: some given back (more than the cut-off packets added), then
# used up again; one address started again; then the whole quota.
run "a top-up for the device" 0 H rule reset-quota "$QR" --give 10mbytes --address $CLIENT4
contains "gave back 10.0 MB"
run "topped up: the device is back" 0 send 1000
send 30000000 >/dev/null 2>&1
run "used up again" 1 send 1000
run "an address that used nothing" 4 H rule reset-quota "$QR" --address 10.88.10.99
run "one address started again" 0 H rule reset-quota "$QR" --address $CLIENT4
run "back again" 0 send 1000
run "reset by hand" 0 H rule reset-quota "$QR"
run "the device is back" 0 send 1000
run "the reset is audited" 0 H audit --action quota.
contains "quota.reset"
contains "quota.give"
# Counted in total now: what the device used carries over; per
# destination can't take it, and says so.
run "within the quota" 0 send 300000
run "to one total" 0 H rule update "$QR" --quota 1mbytes,over
not_contains "quota_restarts"
run "the total carried over" 0 H quotas
contains "e2e quota) *[0-9.]* KB ([0-9]*%)"
run "per destination starts again" 0 H rule update "$QR" --quota 1mbytes,over,per-destination
contains "quota_restarts"
run "the quota rule goes" 0 H rule delete -y "$QR"
# ---- the zone matrix, of the imported ruleset
run "the matrix" 0 H matrix
contains "^firewall *allow"
contains "^dhcp *partly +[0-9]"
contains "dhcp → wan *.*from trusted_ips"
run "LAN to the internet" 0 H matrix dhcp wan
contains "new connections: partly allowed"
contains "allowed *from trusted_ips *rule forward #[0-9]* (Forward from trusted IPs) *[0-9]"
contains "blocked *the rest *forward policy *[0-9]"
not_contains "wg0_net"
run "the internet to the firewall" 0 H matrix wan firewall
contains "blocked *sources that can't come from outside"
contains "allowed *udp/51820"
contains "allowed *raw: icmpv6 type"
# ---- statistics (sampled every 3 seconds here)
run "traffic per device on" 0 H settings set --stats-per-device on --log-drops on
run "the device counters are in the ruleset" 0 R nft list set inet housecarl stats_up_v4
send 500000 >/dev/null 2>&1
run "a connection from the internet, dropped and logged" 1 connect net $WAN4 23
sleep 8
run "statistics are collected" 0 H stats status
contains "Collecting: *yes"
contains "Per device: *on"
run "the summary" 0 H stats --since 1h
contains "^wan *[0-9.]* [KM]B"
contains "Dropped: *[1-9]"
run "traffic per zone over time" 0 H stats traffic --since 1h --step 1m
contains "wan ↓"
run "the devices" 0 H stats top devices --since 1h
contains "^$CLIENT4 (e2e-client) *[0-9.]* KB *[0-9.]* KB"
run "each device once" 0 sh -c "docker exec ${P}-router /w/hc stats top devices --since 1h | grep -c '^$CLIENT4 ' | grep -qx 1"
run "the sources blocked" 0 H stats top blocked --since 1h
contains "^$NET4 *[1-9]"
run "what they tried" 0 H stats top tried --since 1h
contains "^tcp/23"
run "hits per rule" 0 H stats rules --since 1h
contains "input policy *[1-9]"
contains "rule #[0-9]* (deleted) *[1-9]"
run "connections over time" 0 H stats connections --since 1h --step 1m
contains "^[0-9-]* [0-9:]* *[0-9]* *[0-9]*"
# ---- connections and tracing
# A connection that stays open, masqueraded on its way out.
docker exec -d ${P}-client sh -c "sleep 30 | nc $NET4 9000"
sleep 1
run "the connection is tracked" 0 H connections --search "$CLIENT4 :9000"
contains "$CLIENT4:[0-9]* (e2e-client) *$NET4:9000 *as $WAN4"
run "without names" 0 H connections --search "$CLIENT4 :9000" --no-names
contains "$CLIENT4:[0-9]* *$NET4:9000 *as $WAN4"
run "closed" 0 H connections close --search "$CLIENT4 :9000" -y
contains "closed [1-9][0-9]* connection"
run "gone" 0 H connections --search "$CLIENT4 :9000"
not_contains ":9000"
# A block added while traffic flows: it stops new connections, not this
# one, unless it checks every packet.
docker exec -d ${P}-client sh -c "rm -f /tmp/ncdone; yes | nc $NET4 9000; touch /tmp/ncdone"
sleep 1
BR=$(H -o id rule add forward --from $CLIENT4 --service tcp/9000 -a reject --reject-with tcp-reset --comment "e2e block" --first)
run "a block for new connections" 0 test -n "$BR"
sleep 2
run "the open upload carries on" 1 docker exec ${P}-client test -f /tmp/ncdone
run "the block checks every packet now" 0 H rule update "$BR" --every-packet
sleep 2
run "the upload is cut" 0 docker exec ${P}-client test -f /tmp/ncdone
# Tracing a new connection: dropped, by that rule.
docker exec -d ${P}-router sh -c "/w/hc trace --address $CLIENT4 --port 9000 --seconds 6 > /w/trace.out 2>&1"
sleep 2
connect client $NET4 9000 >/dev/null 2>&1
sleep 6
run "the trace" 0 cat "$WORK/trace.out"
contains "$CLIENT4:[0-9]* -> $NET4:9000 tcp syn"
contains "inet housecarl forward: rule forward #$BR (e2e block) -> drop"
contains "=> DROPPED by rule forward #$BR (e2e block)"
run "no trace table is left" 0 R nft list tables
not_contains "housecarl-trace"
run "the block goes" 0 H rule delete -y "$BR"
# ---- the drop log
run "log drops from the internet to the API" 0 H rule add input --in wan --service tcp/8070 -a drop --log --first
connect net $WAN4 8070 >/dev/null 2>&1
sleep 1
run "the log shows it" 0 H log
contains "hc input #.* drop.*198.18.1.10:.*198.18.1.2:8070"
# ---- the flowtable, and devices that appear later
run "a flowtable device that doesn't exist yet" 0 H settings set --flowtable wan --flowtable enp2s0 --flowtable hcft0
contains "flowtable_device_missing"
run "the flowtable has the devices that exist" 0 R nft list flowtables
contains "devices = { enp2s0, wan }"
run "forwarded through the flowtable" 0 connect net $WAN4 8096
contains "jellyfin"
run "the device appears" 0 R ip link add hcft0 type veth peer name hcft1
R ip link set hcft0 up
sleep 4
run "applied again with it" 0 R nft list flowtables
contains "hcft0"
run "logged" 0 R grep -c "interfaces changed: applied the ruleset again" /w/srv.log
run "it comes back as a new interface" 0 R sh -c 'ip link del hcft0 && ip link add hcft0 type veth peer name hcft1 && ip link set hcft0 up'
sleep 4
run "applied again" 0 R sh -c 'grep -c "interfaces changed: applied the ruleset again" /w/srv.log | grep -qx 2'
run "the flowtable still has it" 0 R nft list flowtables
contains "hcft0"
# ---- VLAN sub-interfaces: tagged traffic between the router and the client
R sh -c 'ip link add link enp2s0 name enp2s0.30 type vlan id 30 && ip addr add 10.88.30.1/24 dev enp2s0.30 && ip link set enp2s0.30 up'
docker exec ${P}-client sh -c 'ip link add link eth0 name eth0.30 type vlan id 30 && ip addr add 10.88.30.20/24 dev eth0.30 && ip link set eth0.30 up'
run "a zone of a VLAN" 0 H zone add iot --interface enp2s0.30
run "nothing from the VLAN to the API yet" 1 connect client 10.88.30.1 8070
VR=$(H -o id rule add input --in iot --service tcp/8070 --comment "e2e vlan")
run "a rule for the VLAN's zone" 0 test -n "$VR"
run "the VLAN reaches the API" 0 connect client 10.88.30.1 8070
run "the rule counted it" 0 H rule list input
contains "e2e vlan"
run "the flowtable on the VLAN" 0 H settings set --flowtable wan --flowtable enp2s0 --flowtable hcft0 --flowtable enp2s0.30
run "attached" 0 R nft list flowtables
contains "enp2s0.30"
run "the VLAN rule goes" 0 H rule delete -y "$VR"
run "the VLAN zone goes" 0 H zone delete -y iot
R ip link del enp2s0.30
# ---- ban lists: bans kept from the firewall, a host, or everything
# DNS is open to the internet on the router (tcp/53): a listener answers.
docker exec -d ${P}-router sh -c 'while [ ! -f /tmp/nodns ]; do echo dns | nc -N -l 0.0.0.0 53; done'
sleep 1
run "the router's tcp/53 from the internet" 0 connect net $WAN4 53
contains "^dns$"
run "a ban list for the firewall" 0 H banlist add router --scope firewall
# The lab's internet host shares the router's WAN /64: ban IPv6 addresses
# alone (a /64 would hold the router's own, and is refused).
run "a ban list for jellyfin" 0 H banlist add jf --scope hosts --host $JF4 --host $JF6 --group-v6 128
run "a ban list for everything" 0 H banlist add edge --scope everything --max-time 1h
run "the lists" 0 H banlist list
contains "jf *hosts *10.88.10.57, 2001:db8:7f3d:10::57 *0"
# The firewall's own DNS, for the internet host.
run "ban from the firewall's tcp/53" 0 H ban router $NET4 --tcp dns --reason "e2e"
contains "banned $NET4, tcp/53, in router"
run "the router's DNS is cut for it" 1 connect net $WAN4 53
run "the port forward isn't (scope firewall)" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "the ban in the ruleset" 0 R nft list ruleset
contains "ip saddr . meta l4proto . th dport @ban_[0-9]*_v4_svc counter packets [1-9]"
run "lift it" 0 H unban router $NET4
contains "1 bans lifted"
run "DNS again" 0 connect net $WAN4 53
contains "^dns$"
# fail2ban on jellyfin, with Housecarl's action and a token for its list.
run "a user for jellyfin's fail2ban" 0 H user add f2b-jf --no-password --grant bans:write:jf
R sh -c "/w/hc token create jf --user f2b-jf > /w/f2b.token 2>/dev/null"
run "its token" 0 test -s "$WORK/f2b.token"
# f2b ACTION ip=… port=…: the action's command from contrib/, with the
# jail's values, as fail2ban runs it (on jellyfin).
f2b() {
local conf="$ROOT/contrib/fail2ban/action.d/housecarl.conf" a=$1; shift
local cmd curl
cmd=$(awk -v a="$a" '$0 ~ "^"a" *=" {sub(/^[^=]*= */,""); print; on=1; next} on && /^[ \t]/ {sub(/^[ \t]+/,""); print; next} {on=0}' "$conf" | tr '\n' ' ')
curl=$(sed -n 's/^_curl *= *//p' "$conf")
cmd=${cmd//<_curl>/$curl}
local kv
for kv in "$@" "list=jf" "url=http://$LAN4:8070" "tokenfile=/w/f2b.token" "curlopts=" "protocol=tcp" "bantime=600" "name=jellyfin"; do
cmd=${cmd//<${kv%%=*}>/${kv#*=}}
done
docker exec ${P}-jf sh -c "$cmd"
}
run "fail2ban bans the internet host's https" 0 f2b actionban ip=$NET4 port=https
run "the port forward to jellyfin is cut for it" 1 connect net $WAN4 8096
run "the router's DNS isn't (scope hosts)" 0 connect net $WAN4 53
run "fail2ban bans its IPv6 address" 0 f2b actionban ip=$NET6 port=https
run "IPv6 to jellyfin's 443 is cut" 1 connect net $JF6 443
run "the bans" 0 H bans jf
contains "$NET4 *tcp/443 .*f2b-jf *fail2ban/jellyfin"
contains "$NET6 *tcp/443"
run "never the inside" 22 f2b actionban ip=$CLIENT4 port=https
run "refused, audited" 0 H audit --action ban.add
contains "failure.*inside"
run "fail2ban unbans" 0 f2b actionunban ip=$NET4
run "the port forward works again" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "unbanning what Housecarl already let go is no error" 0 f2b actionunban ip=$NET4
run "and the IPv6 ban" 0 f2b actionunban ip=$NET6
run "jellyfin by IPv6 again" 0 connect net $JF6 443
contains "jellyfin6"
run "the token can't ban elsewhere" 22 f2b actionban ip=$NET4 port=https list=router
# Everything, cutting a connection that is open, and kept over a restart.
R touch /tmp/nodns
connect net $WAN4 53 >/dev/null 2>&1
docker exec -d ${P}-router sh -c 'nc -l 0.0.0.0 53 > /w/held.out'
sleep 1
docker exec -d ${P}-net sh -c "sleep 30 | nc $WAN4 53"
sleep 1
run "an open connection from the internet host" 0 H connections --search "$NET4 :53"
contains "$NET4:[0-9]* *$WAN4:53"
run "ban it from everything" 0 H ban edge $NET4 --time 1d
contains "banned $NET4, every port, in edge (ends in 1h)"
contains "[1-9][0-9]* connections closed"
run "its connection is gone" 0 H connections --search "$NET4 :53"
not_contains ":53"
run "nothing gets through for it" 1 connect net $WAN4 8096
stop
run "restart housecarld" 0 start
run "the ban is back after the restart" 0 R nft list ruleset
contains "$NET4 timeout [0-9hms]* expires"
run "still nothing for it" 1 connect net $WAN4 8096
run "lift it" 0 H unban edge $NET4
run "the ban is out of the ruleset" 0 R nft list ruleset
not_contains "$NET4 timeout"
run "the port forward again" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "the lists go" 0 sh -c "docker exec ${P}-router /w/hc banlist delete -y router && docker exec ${P}-router /w/hc banlist delete -y jf && docker exec ${P}-router /w/hc banlist delete -y edge"
# ---- renaming a zone others refer to
run "rename the imported dhcp zone" 0 H zone update dhcp --name lan
run "its rules follow" 0 H rule list input
contains "lan"
not_contains "dhcp "
run "the ruleset is unchanged by it" 0 R nft list chain inet housecarl input
contains 'iifname "enp2s0" udp dport 67'
# ---- restart and boot
stop
run "the ruleset stays without housecarld" 0 connect net $WAN4 8096
contains "jellyfin"
run "restart housecarld" 0 start
run "after the restart" 0 connect client $WAN4 8096
contains "jellyfin"
run "the boot ruleset" 0 R test -s /w/data/ruleset.nft
R nft flush ruleset
run "nothing forwarded after a flush" 1 connect net $WAN4 8096
run "the boot unit's command restores it" 0 R nft -f /w/data/ruleset.nft
run "forwarded again" 0 connect net $WAN4 8096
contains "jellyfin"
run "the boot ruleset has no flowtable (its devices may not exist yet)" 1 R grep -q flowtable /w/data/ruleset.nft
run "healthz" 0 R curl -sf http://127.0.0.1:8070/healthz
contains '"firewall":"applied"'
# ---- the boot fallback: what the boot unit runs, with the lab's paths
UNIT=$(sed -n 's/^ExecStart=//p' "$ROOT/deploy/systemd/housecarl-ruleset.service" | sed 's|/var/lib/housecarl|/w/data|g; s|/usr/sbin/nft|nft|g')
run "the fallback was saved" 0 R test -s /w/data/ruleset-fallback.nft
R cp /w/data/ruleset.nft /w/ruleset.good
R sh -c 'echo "this is not nftables" >> /w/data/ruleset.nft'
R nft flush ruleset
run "the boot unit fails, and loads the fallback" 1 R sh -c "$UNIT"
contains "the saved ruleset failed to load"
run "the fallback is in force" 0 R nft list table inet housecarl
contains "hc:fallback-input"
run "nothing new gets in from the internet" 1 connect net $WAN4 8096
run "not to the firewall either" 1 connect net $WAN4 8070
: > "$WORK/net.log"
run "the LAN still reaches the internet" 0 connect client $NET4 9000
sleep 1
run "masqueraded" 0 grep -q "$WAN4" "$WORK/net.log"
run "healthz says so" 0 R sh -c 'curl -s -w " %{http_code}" http://127.0.0.1:8070/healthz'
contains '"firewall":"fallback"'
contains " 503"
run "status says so" 0 H status
contains "NOT in force: the boot fallback"
stop
run "housecarld starts" 0 start
run "and puts the real ruleset back" 0 connect net $WAN4 8096
contains "jellyfin"
run "and saves a good boot ruleset" 0 R nft -c -f /w/data/ruleset.nft
run "healthz is happy again" 0 R curl -sf http://127.0.0.1:8070/healthz
contains '"firewall":"applied"'
# ---- backup
run "backup" 0 R /w/housecarld -c /w/housecarl.yaml backup
contains "/w/backups/housecarl-"
echo
echo "RESULT: $PASS passed, $FAIL failed"
[ $FAIL = 0 ]
Reference in new issue View Git Blame Copy Permalink