abrown38andClaude Opus 5.5
34292eeebb
Zones, address sets (IPv4 and IPv6 mixed, dynamic sets with timeouts) and services, ordered rules in the early, input, forward and output chains, port forwards with hairpin NAT and IPv6 pinholes, source NAT, redirects, and switchable built-ins (loopback, conntrack, ICMPv6, antispoofing, MSS clamping, DHCPv6 client, flowtable). Every change is validated, checked with nft -c and applied in one transaction to Housecarl's own table, inet housecarl; changes made over the network roll back unless confirmed, and the confirmed ruleset is loaded at boot by housecarl-ruleset.service. An empty configuration applies nothing. Also: hit counters and the drop log (nflog), 'hc import' of an existing nftables ruleset and 'hc takeover', the hc CLI, the REST API and the web UI, users, scoped tokens (sets:write:NAME), the audit log, the web UI's certificate through Jarl, Debian packages, docs, unit tests with real nft 1.1.3, and an e2e test with real traffic (71 checks). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
220 lines
11 KiB
Bash
Executable File
220 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
# End-to-end test with real traffic: a Debian 13 router container (nftables
|
|
# 1.1.3) between an "internet" network and a LAN, IPv4 and IPv6, runs
|
|
# housecarld. The maintainer's own router ruleset (internal/importer/
|
|
# testdata/router.nft, with lab addresses) is loaded as it is today, imported,
|
|
# taken over, and then checked with connections: port forwards, hairpin NAT,
|
|
# the IPv6 pinhole, masquerading, antispoofing, the drop log, a change on
|
|
# probation that cuts the client off and rolls itself back, a restart and the
|
|
# boot ruleset. Needs Docker; no root on the host. Run from the repository
|
|
# root:
|
|
#
|
|
# make e2e or scripts/e2e.sh
|
|
#
|
|
# KEEP=1 leaves the containers and networks running. Set DOCKER_HOST if
|
|
# Docker isn't at its default socket.
|
|
set -u
|
|
ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
|
P=${E2E_PREFIX:-hce2e}
|
|
IMAGE=${E2E_IMAGE:-housecarl-nft}
|
|
WORK=${WORK:-$(mktemp -d "${TMPDIR:-/tmp}/housecarl-e2e.XXXXXX")}
|
|
docker info >/dev/null 2>&1 || { echo "Docker isn't reachable (set DOCKER_HOST?)"; exit 2; }
|
|
|
|
(cd "$ROOT" && CGO_ENABLED=0 go build -o "$WORK/housecarld" ./cmd/housecarld && CGO_ENABLED=0 go build -o "$WORK/hc" ./cmd/hc) || exit 1
|
|
# The image: Debian 13 with nftables and network tools. 'docker build'
|
|
# can't reach the network on some hosts, so it is made with run + commit.
|
|
if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
|
|
docker rm -f ${P}-build >/dev/null 2>&1
|
|
docker run --name ${P}-build debian:trixie sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get update -qq && apt-get install -y -qq --no-install-recommends nftables iproute2 iputils-ping netcat-openbsd curl ca-certificates >/dev/null 2>&1 && rm -rf /var/lib/apt/lists/*' || exit 1
|
|
docker commit ${P}-build "$IMAGE" >/dev/null && docker rm ${P}-build >/dev/null || exit 1
|
|
fi
|
|
|
|
cleanup() {
|
|
if [ "${KEEP:-}" = 1 ]; then echo "left running: containers ${P}-*, networks ${P}-*, files in $WORK"; return; fi
|
|
docker exec ${P}-router sh -c 'rm -rf /w/data /w/backups /w/*.log' >/dev/null 2>&1
|
|
docker rm -f $(docker ps -aq --filter "name=^${P}-") >/dev/null 2>&1
|
|
docker network rm ${P}-wan ${P}-lan ${P}-wg >/dev/null 2>&1
|
|
rm -rf "$WORK"
|
|
}
|
|
trap cleanup EXIT
|
|
docker rm -f $(docker ps -aq --filter "name=^${P}-") >/dev/null 2>&1
|
|
docker network rm ${P}-wan ${P}-lan ${P}-wg >/dev/null 2>&1
|
|
|
|
# The lab: "the internet" 198.18.1.0/24 and 2001:db8:1::/64; the LAN
|
|
# 10.88.10.0/24 and 2001:db8:7f3d:10::/64 (the /48 is the delegated
|
|
# prefix); a WireGuard-like network 10.88.50.0/24. The router has .1 on its
|
|
# inside networks (Docker's gateway moves to .254).
|
|
docker network create --ipv6 --subnet 198.18.1.0/24 --gateway 198.18.1.254 --subnet 2001:db8:1::/64 --gateway 2001:db8:1::fe ${P}-wan >/dev/null || exit 1
|
|
docker network create --ipv6 --subnet 10.88.10.0/24 --gateway 10.88.10.254 --subnet 2001:db8:7f3d:10::/64 --gateway 2001:db8:7f3d:10::fe ${P}-lan >/dev/null || exit 1
|
|
docker network create --subnet 10.88.50.0/24 --gateway 10.88.50.254 ${P}-wg >/dev/null || exit 1
|
|
WAN4=198.18.1.2 WAN6=2001:db8:1::2 NET4=198.18.1.10 NET6=2001:db8:1::10
|
|
LAN4=10.88.10.1 LAN6=2001:db8:7f3d:10::1 JF4=10.88.10.57 JF6=2001:db8:7f3d:10::57 CLIENT4=10.88.10.20
|
|
|
|
sysctls="--sysctl net.ipv4.ip_forward=1 --sysctl net.ipv6.conf.all.forwarding=1 --sysctl net.ipv6.conf.all.keep_addr_on_down=1 --sysctl net.ipv6.conf.default.keep_addr_on_down=1"
|
|
docker run -d --name ${P}-router --hostname router --cap-add NET_ADMIN $sysctls \
|
|
--network ${P}-wan --ip $WAN4 --ip6 $WAN6 -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1
|
|
docker network connect --ip $LAN4 --ip6 $LAN6 ${P}-lan ${P}-router || exit 1
|
|
docker network connect --ip 10.88.50.1 ${P}-wg ${P}-router || exit 1
|
|
# The router's interfaces get the names of the real one: wan, enp2s0, wg0
|
|
# (attached in that order: eth0, eth1, eth2).
|
|
docker exec ${P}-router sh -c 'for p in eth0:wan eth1:enp2s0 eth2:wg0; do i=${p%:*} n=${p#*:}; ip link set $i down && ip link set $i name $n && ip link set $n up; done
|
|
ip route replace default via 198.18.1.254' || exit 1
|
|
# Hosts: one on the internet, a server and a client on the LAN; inside
|
|
# hosts route through the router, the internet reaches the delegated
|
|
# prefix through it.
|
|
host() { # NAME NETWORK IP4 IP6 GATEWAY4 GATEWAY6
|
|
docker run -d --name ${P}-$1 --cap-add NET_ADMIN --sysctl net.ipv6.conf.all.keep_addr_on_down=1 --network ${P}-$2 --ip $3 ${4:+--ip6 $4} -v "$WORK:/w" "$IMAGE" sleep infinity >/dev/null || exit 1
|
|
docker exec ${P}-$1 sh -c "ip route replace default via $5 ${6:+&& ip -6 route replace default via $6}" || exit 1
|
|
}
|
|
host net wan $NET4 $NET6 198.18.1.254
|
|
docker exec ${P}-net ip -6 route add 2001:db8:7f3d::/48 via $WAN6
|
|
host jf lan $JF4 $JF6 $LAN4 $LAN6
|
|
host client lan $CLIENT4 2001:db8:7f3d:10::20 $LAN4 $LAN6
|
|
# Servers: jellyfin on 443 (IPv4 and IPv6) and 22, a listener on the
|
|
# internet that logs who connects.
|
|
docker exec -d ${P}-jf sh -c 'while true; do echo jellyfin | nc -N -l 0.0.0.0 443; done'
|
|
docker exec -d ${P}-jf sh -c 'while true; do echo jellyfin6 | nc -6 -N -l :: 443; done'
|
|
docker exec -d ${P}-jf sh -c 'while true; do echo ssh | nc -6 -N -l :: 22; done'
|
|
docker exec -d ${P}-net sh -c 'nc -lkvn 0.0.0.0 9000 2>>/w/net.log >/dev/null'
|
|
|
|
PASS=0; FAIL=0
|
|
ok() { echo "PASS: $1"; PASS=$((PASS+1)); }
|
|
bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); }
|
|
# run DESC EXPECTED_RC CMD...: runs a command, checks its exit code and
|
|
# keeps the output in $OUT.
|
|
run() {
|
|
local desc=$1 want=$2; shift 2
|
|
OUT=$("$@" 2>&1); local rc=$?
|
|
if [ "$rc" = "$want" ]; then ok "$desc (rc=$rc)"; else bad "$desc: rc=$rc want $want"; echo "$OUT" | sed 's/^/ /'; fi
|
|
}
|
|
contains() { if echo "$OUT" | grep -q -- "$1"; then ok "output has '$1'"; else bad "output lacks '$1'"; echo "$OUT" | sed 's/^/ /'; fi; }
|
|
not_contains() { if echo "$OUT" | grep -q -- "$1"; then bad "output has '$1'"; echo "$OUT" | sed 's/^/ /'; else ok "output lacks '$1'"; fi; }
|
|
R() { docker exec ${P}-router "$@"; }
|
|
H() { docker exec ${P}-router /w/hc "$@"; }
|
|
# connect FROM HOST PORT: what the server says, within 3 seconds.
|
|
connect() { docker exec ${P}-$1 nc -w 3 "$2" "$3" </dev/null; }
|
|
|
|
cat > "$WORK/housecarl.yaml" <<'EOF'
|
|
listen: ["0.0.0.0:8070"]
|
|
database: /w/data/housecarl.db
|
|
ruleset: /w/data/ruleset.nft
|
|
backup: {dir: /w/backups}
|
|
log: {level: debug}
|
|
EOF
|
|
start() { R sh -c '(/w/housecarld -c /w/housecarl.yaml serve >> /w/srv.log 2>&1 &); for i in $(seq 1 30); do [ -S /run/housecarl/housecarl.sock ] && exit 0; sleep 0.3; done; exit 1'; }
|
|
|
|
# ---- today's router: the hand-written ruleset
|
|
|
|
sed -e 's/217\.155\.8\.194/198.18.1.2/g' -e 's/10\.0\.10\./10.88.10./g' -e 's/10\.50\.50\./10.88.50./g' \
|
|
-e 's/2a02:8013:7f3d/2001:db8:7f3d/g' "$ROOT/internal/importer/testdata/router.nft" > "$WORK/router.nft"
|
|
run "today's ruleset loads" 0 R nft -f /w/router.nft
|
|
R nft add table ip other
|
|
run "the port forward works under it" 0 connect net $WAN4 8096
|
|
contains "jellyfin"
|
|
|
|
# ---- import and take over
|
|
|
|
run "init" 0 R sh -c 'echo secret-pass-123 | /w/housecarld -c /w/housecarl.yaml init --password-stdin'
|
|
run "start housecarld" 0 start
|
|
run "status" 0 H status
|
|
contains "Other tables:.*inet main"
|
|
run "two tables: say which" 6 H import --defines /w/router.nft -n
|
|
contains "several tables"
|
|
run "dry run of the import" 0 H import --table "inet main" --defines /w/router.nft -n
|
|
contains "5 port forwards, 2 source NAT rules"
|
|
contains "dry run: nothing was changed"
|
|
run "import" 0 H import --table "inet main" --defines /w/router.nft
|
|
contains "applied and saved"
|
|
run "the ruleset is in force" 0 R nft list table inet housecarl
|
|
contains "dnat ip to 10.88.10.57:443"
|
|
run "zones" 0 H zone list
|
|
contains "wan.*wan.*yes"
|
|
contains "dhcp.*enp2s0"
|
|
run "port forwards" 0 H forward list
|
|
contains "jellyfin.*198.18.1.2 tcp/8096.*10.88.10.57:443"
|
|
run "take over the old table" 0 H takeover "inet main" -y
|
|
run "only Housecarl's table and the other one remain" 0 R nft list tables
|
|
not_contains "inet main"
|
|
contains "ip other"
|
|
|
|
# ---- traffic
|
|
|
|
run "port forward from the internet" 0 connect net $WAN4 8096
|
|
contains "^jellyfin$"
|
|
run "hairpin: the public address from the LAN" 0 connect client $WAN4 8096
|
|
contains "^jellyfin$"
|
|
# The IPv6 rule stays a forward rule: jellyfin and haproxy both use inside
|
|
# port 443, so the importer doesn't guess whose pinhole it is.
|
|
run "the IPv6 rule" 0 H rule list forward
|
|
contains "to 2001:db8:7f3d:10::57 tcp/443 *accept"
|
|
run "IPv6 to the host's port 443 from the internet" 0 connect net $JF6 443
|
|
contains "jellyfin6"
|
|
run "IPv6 to another port of the host is dropped" 1 connect net $JF6 22
|
|
run "masquerade: the LAN reaches the internet as the router" 0 docker exec ${P}-client sh -c "echo hi | nc -N -w 3 $NET4 9000"
|
|
sleep 1
|
|
run "the internet saw the router's address" 0 cat "$WORK/net.log"
|
|
contains "$WAN4"
|
|
not_contains "$CLIENT4"
|
|
run "nothing else gets in from the internet" 1 connect net $JF4 443
|
|
# Antispoofing: a source that can't come from the internet.
|
|
docker exec ${P}-net ip addr add 10.9.9.9/32 dev eth0
|
|
docker exec ${P}-net ping -c 2 -W 1 -I 10.9.9.9 $WAN4 >/dev/null 2>&1
|
|
run "counters" 0 H counters
|
|
contains "b-antispoof *[1-9]"
|
|
|
|
# ---- the API from the LAN, on probation
|
|
|
|
# Allow the API from the trusted networks (through the local socket: no
|
|
# probation).
|
|
APIRULE=$(H -o id rule add input --from trusted_ips --service tcp/8070 --comment "Housecarl API")
|
|
run "the API rule was added" 0 test -n "$APIRULE"
|
|
run "probation of 10 seconds" 0 H settings set --probation 10s
|
|
TOKEN=$(R /w/housecarld -c /w/housecarl.yaml user token admin e2e 2>/dev/null | tail -1)
|
|
C() { docker exec ${P}-client /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" "$@"; }
|
|
run "the API from the LAN" 0 C whoami
|
|
run "a change over the network is on probation" 0 C zone add guests --interface guest0
|
|
contains "applied on probation"
|
|
run "confirmed" 0 C confirm
|
|
contains "confirmed 1 change;"
|
|
run "a change that cuts the client off" 0 C rule disable "$APIRULE"
|
|
contains "on probation"
|
|
run "the client is cut off (a timeout)" 124 docker exec ${P}-client timeout 3 /w/hc --config /w/cli.yaml --server http://$LAN4:8070 --token "$TOKEN" whoami
|
|
sleep 12
|
|
run "rolled back by itself: the client is back" 0 C whoami
|
|
run "the rule is enabled again" 0 H rule show "$APIRULE"
|
|
not_contains "disabled"
|
|
run "the rollback is in the history" 0 H history list
|
|
contains "rollback"
|
|
|
|
# ---- the drop log
|
|
|
|
run "log drops from the internet to the API" 0 H rule add input --in wan --service tcp/8070 -a drop --log --first
|
|
connect net $WAN4 8070 >/dev/null 2>&1
|
|
sleep 1
|
|
run "the log shows it" 0 H log
|
|
contains "hc input #.* drop.*198.18.1.10:.*198.18.1.2:8070"
|
|
|
|
# ---- restart and boot
|
|
|
|
docker exec ${P}-router sh -c 'kill $(pidof housecarld)'; sleep 1
|
|
run "the ruleset stays without housecarld" 0 connect net $WAN4 8096
|
|
contains "jellyfin"
|
|
run "restart housecarld" 0 start
|
|
run "after the restart" 0 connect client $WAN4 8096
|
|
contains "jellyfin"
|
|
run "the boot ruleset" 0 R test -s /w/data/ruleset.nft
|
|
R nft flush ruleset
|
|
run "nothing forwarded after a flush" 1 connect net $WAN4 8096
|
|
run "the boot unit's command restores it" 0 R nft -f /w/data/ruleset.nft
|
|
run "forwarded again" 0 connect net $WAN4 8096
|
|
contains "jellyfin"
|
|
|
|
# ---- backup
|
|
|
|
run "backup" 0 R /w/housecarld -c /w/housecarl.yaml backup
|
|
contains "/w/backups/housecarl-"
|
|
|
|
echo
|
|
echo "RESULT: $PASS passed, $FAIL failed"
|
|
[ $FAIL = 0 ]
|