adam/housecarl
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity
Files
fc390eb57cc2517b2184b9e46925e62366c289d6
housecarl/scripts/e2e.sh
T
abrown38andClaude Opus 5.5 2880f7ddc9 Add changesets, bandwidth limits and quotas; generic examples 
Changesets, as in Jarl: configuration changes are operations that can be
staged in a changeset, checked as they are added (validation and nft -c),
reviewed with the diff of the ruleset, and applied together as one
history entry, then on probation like any change made over the network.
The web UI collects edits for review by default (pending rows, a bar with
Review and Apply now, a Changes page, a switch for live edits); hc stages
with -C ID or HOUSECARL_CHANGESET (checked like JARL_CHANGESET) and has
hc changeset create/list/show/diff/apply/discard/remove. Undo, import and
takeover refuse a changeset. --dry-run shows the ruleset's diff.

Bandwidth and quotas: rate limits in bytes, per source or destination;
quotas per rule or per address that reset daily, weekly, monthly on a
day, or by hand, with their usage carried across reloads, saved for
reboots, shown by hc quotas and on the rules page, and reset with hc rule
reset-quota. Rules that count traffic are checked before established
connections are accepted (so they see every packet) and can't accept.
Per-address limits group IPv6 by /64 by default (--group-v4/-v6).
Per-source rate meters use update. The importer turns meters, byte rates
and quotas into these fields.

The examples in the web UI, hc's help and the docs use documentation
addresses and generic names instead of the maintainer's network.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 11:18:54 +01:00

282 lines
14 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# End-to-end test with real traffic: a Debian 13 router container (nftables
# 1.1.3) between an "internet" network and a LAN, IPv4 and IPv6, runs
# housecarld. The maintainer's own router ruleset (internal/importer/
# testdata/router.nft, with lab addresses) is loaded as it is today, imported,
# taken over, and then checked with connections: port forwards, hairpin NAT,
# the IPv6 pinhole, masquerading, antispoofing, the drop log, a change on
# probation that cuts the client off and rolls itself back, a restart and the
# boot ruleset. Needs Docker; no root on the host. Run from the repository
# root:
#
# make e2e or scripts/e2e.sh
#
# KEEP=1 leaves the containers and networks running. Set DOCKER_HOST if
# Docker isn't at its default socket.
set -u
ROOT=$(cd "$(dirname "$0")/.." && pwd)
P=${E2E_PREFIX:-hce2e}
IMAGE=${E2E_IMAGE:-housecarl-nft}
WORK=${WORK:-$(mktemp -d "${TMPDIR:-/tmp}/housecarl-e2e.XXXXXX")}
docker info >/dev/null 2>&1 || { echo "Docker isn't reachable (set DOCKER_HOST?)"; exit 2; }
(cd "$ROOT" && CGO_ENABLED=0 go build -o "$WORK/housecarld" ./cmd/housecarld && CGO_ENABLED=0 go build -o "$WORK/hc" ./cmd/hc) || exit 1
# The image: Debian 13 with nftables and network tools. 'docker build'
# can't reach the network on some hosts, so it is made with run + commit.
if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
docker rm -f ${P}-build >/dev/null 2>&1
docker run --name ${P}-build debian:trixie sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get update -qq && apt-get install -y -qq --no-install-recommends nftables iproute2 iputils-ping netcat-openbsd curl ca-certificates >/dev/null 2>&1 && rm -rf /var/lib/apt/lists/*' || exit 1
docker commit ${P}-build "$IMAGE" >/dev/null && docker rm ${P}-build >/dev/null || exit 1
fi
cleanup() {
if [ "${KEEP:-}" = 1 ]; then echo "left running: containers ${P}-*, networks ${P}-*, files in $WORK"; return; fi
docker exec ${P}-router sh -c 'rm -rf /w/data /w/backups /w/*.log' >/dev/null 2>&1
docker rm -f $(docker ps -aq --filter "name=^${P}-") >/dev/null 2>&1
docker network rm ${P}-wan ${P}-lan ${P}-wg >/dev/null 2>&1
rm -rf "$WORK"
}
trap cleanup EXIT
docker rm -f $(docker ps -aq --filter "name=^${P}-") >/dev/null 2>&1
docker network rm ${P}-wan ${P}-lan ${P}-wg >/dev/null 2>&1
# The lab: "the internet" 198.18.1.0/24 and 2001:db8:1::/64; the LAN
# 10.88.10.0/24 and 2001:db8:7f3d:10::/64 (the /48 is the delegated
# prefix); a WireGuard-like network 10.88.50.0/24. The router has .1 on its
# inside networks (Docker's gateway moves to .254).
docker network create --ipv6 --subnet 198.18.1.0/24 --gateway 198.18.1.254 --subnet 2001:db8:1::/64 --gateway 2001:db8:1::fe ${P}-wan >/dev/null || exit 1
docker network create --ipv6 --subnet 10.88.10.0/24 --gateway 10.88.10.254 --subnet 2001:db8:7f3d:10::/64 --gateway 2001:db8:7f3d:10::fe ${P}-lan >/dev/null || exit 1
docker network create --subnet 10.88.50.0/24 --gateway 10.88.50.254 ${P}-wg >/dev/null || exit 1
WAN4=198.18.1.2 WAN6=2001:db8:1::2 NET4=198.18.1.10 NET6=2001:db8:1::10
LAN4=10.88.10.1 LAN6=2001:db8:7f3d:10::1 JF4=10.88.10.57 JF6=2001:db8:7f3d:10::57 CLIENT4=10.88.10.20
sysctls="--sysctl net.ipv4.ip_forward=1 --sysctl net.ipv6.conf.all.forwarding=1 --sysctl net.ipv6.conf.all.keep_addr_on_down=1 --sysctl net.ipv6.conf.default.keep_addr_on_down=1"
docker run -d --name ${P}-router --hostname router --cap-add NET_ADMIN $sysctls \
--network ${P}-wan --ip $WAN4 --ip6 $WAN6 -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1
docker network connect --ip $LAN4 --ip6 $LAN6 ${P}-lan ${P}-router || exit 1
docker network connect --ip 10.88.50.1 ${P}-wg ${P}-router || exit 1
# The router's interfaces get the names of the real one: wan, enp2s0, wg0
# (attached in that order: eth0, eth1, eth2).
docker exec ${P}-router sh -c 'for p in eth0:wan eth1:enp2s0 eth2:wg0; do i=${p%:*} n=${p#*:}; ip link set $i down && ip link set $i name $n && ip link set $n up; done
ip route replace default via 198.18.1.254' || exit 1
# Hosts: one on the internet, a server and a client on the LAN; inside
# hosts route through the router, the internet reaches the delegated
# prefix through it.
host() { # NAME NETWORK IP4 IP6 GATEWAY4 GATEWAY6
docker run -d --name ${P}-$1 --cap-add NET_ADMIN --sysctl net.ipv6.conf.all.keep_addr_on_down=1 --network ${P}-$2 --ip $3 ${4:+--ip6 $4} -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1
docker exec ${P}-$1 sh -c "ip route replace default via $5 ${6:+&& ip -6 route replace default via $6}" || exit 1
}
host net wan $NET4 $NET6 198.18.1.254
docker exec ${P}-net ip -6 route add 2001:db8:7f3d::/48 via $WAN6
host jf lan $JF4 $JF6 $LAN4 $LAN6
host client lan $CLIENT4 2001:db8:7f3d:10::20 $LAN4 $LAN6
# Servers: jellyfin on 443 (IPv4 and IPv6) and 22, a listener on the
# internet that logs who connects.
docker exec -d ${P}-jf sh -c 'while true; do echo jellyfin | nc -N -l 0.0.0.0 443; done'
docker exec -d ${P}-jf sh -c 'while true; do echo jellyfin6 | nc -6 -N -l :: 443; done'
docker exec -d ${P}-jf sh -c 'while true; do echo ssh | nc -6 -N -l :: 22; done'
docker exec -d ${P}-net sh -c 'nc -lkvn 0.0.0.0 9000 2>>/w/net.log >/dev/null'
PASS=0; FAIL=0
ok() { echo "PASS: $1"; PASS=$((PASS+1)); }
bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); }
# run DESC EXPECTED_RC CMD...: runs a command, checks its exit code and
# keeps the output in $OUT.
run() {
local desc=$1 want=$2; shift 2
OUT=$("$@" 2>&1); local rc=$?
if [ "$rc" = "$want" ]; then ok "$desc (rc=$rc)"; else bad "$desc: rc=$rc want $want"; echo "$OUT" | sed 's/^/ /'; fi
}
contains() { if echo "$OUT" | grep -q -- "$1"; then ok "output has '$1'"; else bad "output lacks '$1'"; echo "$OUT" | sed 's/^/ /'; fi; }
not_contains() { if echo "$OUT" | grep -q -- "$1"; then bad "output has '$1'"; echo "$OUT" | sed 's/^/ /'; else ok "output lacks '$1'"; fi; }
R() { docker exec ${P}-router "$@"; }
H() { docker exec ${P}-router /w/hc "$@"; }
# connect FROM HOST PORT: what the server says, within 3 seconds.
connect() { docker exec ${P}-$1 nc -w 3 "$2" "$3" </dev/null; }
cat > "$WORK/housecarl.yaml" <<'EOF'
listen: ["0.0.0.0:8070"]
database: /w/data/housecarl.db
ruleset: /w/data/ruleset.nft
backup: {dir: /w/backups}
log: {level: debug}
EOF
start() { R sh -c '(/w/housecarld -c /w/housecarl.yaml serve >> /w/srv.log 2>&1 &); for i in $(seq 1 30); do [ -S /run/housecarl/housecarl.sock ] && exit 0; sleep 0.3; done; exit 1'; }
# ---- today's router: the hand-written ruleset
sed -e 's/217\.155\.8\.194/198.18.1.2/g' -e 's/10\.0\.10\./10.88.10./g' -e 's/10\.50\.50\./10.88.50./g' \
-e 's/2a02:8013:7f3d/2001:db8:7f3d/g' "$ROOT/internal/importer/testdata/router.nft" > "$WORK/router.nft"
run "today's ruleset loads" 0 R nft -f /w/router.nft
R nft add table ip other
run "the port forward works under it" 0 connect net $WAN4 8096
contains "jellyfin"
# ---- import and take over
run "init" 0 R sh -c 'echo secret-pass-123 | /w/housecarld -c /w/housecarl.yaml init --password-stdin'
run "start housecarld" 0 start
run "status" 0 H status
contains "Other tables:.*inet main"
run "two tables: say which" 6 H import --defines /w/router.nft -n
contains "several tables"
run "dry run of the import" 0 H import --table "inet main" --defines /w/router.nft -n
contains "5 port forwards, 2 source NAT rules"
contains "dry run: nothing was changed"
run "import" 0 H import --table "inet main" --defines /w/router.nft
contains "applied and saved"
run "the ruleset is in force" 0 R nft list table inet housecarl
contains "dnat ip to 10.88.10.57:443"
run "zones" 0 H zone list
contains "wan.*wan.*yes"
contains "dhcp.*enp2s0"
run "port forwards" 0 H forward list
contains "jellyfin.*198.18.1.2 tcp/8096.*10.88.10.57:443"
run "take over the old table" 0 H takeover "inet main" -y
run "only Housecarl's table and the other one remain" 0 R nft list tables
not_contains "inet main"
contains "ip other"
# ---- traffic
run "port forward from the internet" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "hairpin: the public address from the LAN" 0 connect client $WAN4 8096
contains "^jellyfin$"
# The IPv6 rule stays a forward rule: jellyfin and haproxy both use inside
# port 443, so the importer doesn't guess whose pinhole it is.
run "the IPv6 rule" 0 H rule list forward
contains "to 2001:db8:7f3d:10::57 tcp/443 *accept"
run "IPv6 to the host's port 443 from the internet" 0 connect net $JF6 443
contains "jellyfin6"
run "IPv6 to another port of the host is dropped" 1 connect net $JF6 22
run "masquerade: the LAN reaches the internet as the router" 0 docker exec ${P}-client sh -c "echo hi | nc -N -w 3 $NET4 9000"
sleep 1
run "the internet saw the router's address" 0 cat "$WORK/net.log"
contains "$WAN4"
not_contains "$CLIENT4"
run "nothing else gets in from the internet" 1 connect net $JF4 443
# Antispoofing: a source that can't come from the internet.
docker exec ${P}-net ip addr add 10.9.9.9/32 dev eth0
docker exec ${P}-net ping -c 2 -W 1 -I 10.9.9.9 $WAN4 >/dev/null 2>&1
run "counters" 0 H counters
contains "b-antispoof *[1-9]"
# ---- the API from the LAN, on probation
# Allow the API from the trusted networks (through the local socket: no
# probation).
APIRULE=$(H -o id rule add input --from trusted_ips --service tcp/8070 --comment "Housecarl API")
run "the API rule was added" 0 test -n "$APIRULE"
run "probation of 10 seconds" 0 H settings set --probation 10s
TOKEN=$(R /w/housecarld -c /w/housecarl.yaml user token admin e2e 2>/dev/null | tail -1)
C() { docker exec ${P}-client /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" "$@"; }
run "the API from the LAN" 0 C whoami
run "a change over the network is on probation" 0 C zone add guests --interface guest0
contains "applied on probation"
run "confirmed" 0 C confirm
contains "confirmed 1 change;"
run "a change that cuts the client off" 0 C rule disable "$APIRULE"
contains "on probation"
run "the client is cut off (a timeout)" 124 docker exec ${P}-client timeout 3 /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" whoami
sleep 12
run "rolled back by itself: the client is back" 0 C whoami
run "the rule is enabled again" 0 H rule show "$APIRULE"
not_contains "disabled"
run "the rollback is in the history" 0 H history list
contains "rollback"
# ---- changesets: staged, reviewed, applied together
CS=$(C changeset create "e2e batch" -q)
run "a changeset" 0 test -n "$CS"
CE() { docker exec -e HOUSECARL_CHANGESET=$CS ${P}-client /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" "$@"; }
run "staged from HOUSECARL_CHANGESET" 0 CE set add e2e_block 198.51.100.7
contains "staged in changeset $CS as operation 1"
contains "from HOUSECARL_CHANGESET"
run "a rule naming the staged set" 0 CE rule add input --from e2e_block --service ssh -a drop
contains "operation 2"
run "disabling the jellyfin forward, staged" 0 CE forward update jellyfin --disabled
contains "operation 3"
run "nothing is live yet: the forward still works" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "the set isn't there yet" 4 H set show e2e_block
run "the changeset's diff" 0 CE changeset diff
contains "^+.*ip saddr @e2e_block_v4 tcp dport 22 counter drop"
contains "^-.*dnat ip to 10.88.10.57:443"
contains "valid: nftables accepts"
run "remove the forward operation" 0 CE changeset remove 3
contains "has 2 operations"
run "the diff without it" 0 CE changeset diff
not_contains "dnat ip to 10.88.10.57"
run "applied together" 0 CE changeset apply
contains "applied on probation"
contains "HOUSECARL_CHANGESET still names changeset $CS"
run "one history entry" 0 H history list
contains "e2e batch"
run "confirmed" 0 C confirm
run "the set is there now" 0 H set show e2e_block
run "the forward still works" 0 connect net $WAN4 8096
contains "^jellyfin$"
run "a stale HOUSECARL_CHANGESET is refused" 1 CE zone add stale -i stale0
contains "is applied"
CS2=$(H changeset create -q)
run "an undo can't be staged" 6 H -C "$CS2" history undo 1
contains "can't be staged"
run "discarded" 0 H changeset discard "$CS2"
run "a dry run shows the ruleset's diff" 0 H -n set add e2e_dry 10.1.2.3
contains "^+.*set e2e_dry_v4"
# ---- quotas: an allowance per device, kept across a restart, reset
QR=$(H -o id rule add forward --from $CLIENT4 --service tcp/9000 --quota 1mbytes,over,per-source -a drop --first --comment "e2e quota")
run "a per-device quota" 0 test -n "$QR"
send() { docker exec ${P}-client sh -c "head -c $1 /dev/zero | nc -N -w 3 $NET4 9000"; }
run "within the quota" 0 send 300000
send 2000000 >/dev/null 2>&1
run "used up: the device is cut off" 1 send 1000
run "the usage" 0 H quotas
contains "e2e quota.*1.0 MB each.*by hand"
contains "$CLIENT4 *1\.[0-9] MB"
docker exec ${P}-router sh -c 'kill $(pidof housecarld)'; sleep 1; start
run "after a restart, still used up" 1 send 1000
run "the usage after a restart" 0 H quotas
contains "$CLIENT4 *1\.[0-9] MB"
run "reset by hand" 0 H rule reset-quota "$QR"
run "the device is back" 0 send 1000
run "the reset is audited" 0 H audit --action quota.
contains "quota.reset"
run "the quota rule goes" 0 H rule delete -y "$QR"
# ---- the drop log
run "log drops from the internet to the API" 0 H rule add input --in wan --service tcp/8070 -a drop --log --first
connect net $WAN4 8070 >/dev/null 2>&1
sleep 1
run "the log shows it" 0 H log
contains "hc input #.* drop.*198.18.1.10:.*198.18.1.2:8070"
# ---- restart and boot
docker exec ${P}-router sh -c 'kill $(pidof housecarld)'; sleep 1
run "the ruleset stays without housecarld" 0 connect net $WAN4 8096
contains "jellyfin"
run "restart housecarld" 0 start
run "after the restart" 0 connect client $WAN4 8096
contains "jellyfin"
run "the boot ruleset" 0 R test -s /w/data/ruleset.nft
R nft flush ruleset
run "nothing forwarded after a flush" 1 connect net $WAN4 8096
run "the boot unit's command restores it" 0 R nft -f /w/data/ruleset.nft
run "forwarded again" 0 connect net $WAN4 8096
contains "jellyfin"
# ---- backup
run "backup" 0 R /w/housecarld -c /w/housecarl.yaml backup
contains "/w/backups/housecarl-"
echo
echo "RESULT: $PASS passed, $FAIL failed"
[ $FAIL = 0 ]
Reference in new issue View Git Blame Copy Permalink