Release 0.1.1-rc.13

Housecarl 0.1.1's thirteenth release candidate. Since rc.12: quota usage
is saved after every change. make test and make e2e (304 checks) pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -8,6 +8,9 @@ All notable changes to Housecarl are documented here. The format follows
 
 ### Added
 
+- **Quota usage as a share.** What a quota used shows its percentage
+  (`12.3 GB of 50.0 GB used (24%)`, each address's in the quota
+  dialog), on the Rules page, the dashboard and in `hc quotas`.
 - **Reset one address's quota, or give some back.** `hc rule
   reset-quota ID --address ADDR` starts only that device's allowance
   again; `--give 5gbytes` takes that much off what was used instead (one
@@ -93,6 +96,13 @@ All notable changes to Housecarl are documented here. The format follows
   the old entries behind. Now a total starts at the addresses' sum,
   wider groups add up their addresses, and what can't be carried starts
   again with a warning (`quota_restarts`) on the change.
+- What the quotas used is saved after every change, not only every five
+  minutes: a reboot soon after a change that counts a quota differently
+  could bring back its usage counted the old way.
+- Statistics' Quotas card linked every entry to the Rules page, not to
+  its rule; each now opens its rule, named. Per-address quotas were
+  missing from it (and from the `quota:rN` series): they count their
+  addresses' sum.
 - `hc stats top devices` listed each device twice (once for what it
   received, once for what it sent); it is one row with both again.
 - `hc rule add --before/--after/--first` made two changes (create at the

README.md

@@ -123,7 +123,7 @@ The web UI is on `http://127.0.0.1:8070/` (see `listen` in
 
 ## Status
 
-Housecarl 0.1.1 is in testing (0.1.1-rc.11): zones, address sets and
+Housecarl 0.1.1 is in testing (0.1.1-rc.13): zones, address sets and
 services with ordered rules, IPv4 and IPv6, port forwards with hairpin NAT
 and IPv6 pinholes, source NAT and redirects, built-ins, probation for
 network changes (confirmed by the client that made them), the ruleset

VERSION

@@ -1 +1 @@
-0.1.1-rc.11
+0.1.1-rc.13

cmd/hc/firewall.go

@@ -322,9 +322,13 @@ func quotasCmd() *cobra.Command {
 					if q.Quota.PerSource || q.Quota.PerDestination {
 						of += " each"
 					}
-					fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", q.Rule, humanBytes(q.Used), of, resets, next)
+					used := humanBytes(q.Used)
+					if !q.Quota.PerSource && !q.Quota.PerDestination {
+						used += " (" + percent(q.Used, q.Quota.Bytes) + ")"
+					}
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", q.Rule, used, of, resets, next)
 					for _, a := range q.Addresses {
-						fmt.Fprintf(w, "  %s\t%s\t\t\t\n", named(names, a.Address, a.Address), humanBytes(a.Used))
+						fmt.Fprintf(w, "  %s\t%s (%s)\t\t\t\n", named(names, a.Address, a.Address), humanBytes(a.Used), percent(a.Used, q.Quota.Bytes))
 					}
 				}
 			})
@@ -332,6 +336,17 @@ func quotasCmd() *cobra.Command {
 	}
 }
 
+// percent writes used as a share of total: "25%", "<1%" for a little.
+func percent(used, total int64) string {
+	if total <= 0 || used <= 0 {
+		return "0%"
+	}
+	if used*100 < total {
+		return "<1%"
+	}
+	return fmt.Sprintf("%d%%", used*100/total)
+}
+
 func logCmd() *cobra.Command {
 	var follow bool
 	var limit int

cmd/hc/main_test.go

@@ -53,6 +53,11 @@ func TestLimitAndDescribe(t *testing.T) {
 			t.Errorf("describe(%+v, %s) = %q, want %q", c.m, c.chain, got, c.want)
 		}
 	}
+	for used, want := range map[int64]string{0: "0%", 5: "<1%", 250: "25%", 1000: "100%", 1300: "130%"} {
+		if got := percent(used, 1000); got != want {
+			t.Errorf("percent(%d, 1000) = %q, want %q", used, got, want)
+		}
+	}
 	if got := editList([]string{"a", "b", "c"}, []string{"d"}, []string{"b"}); len(got) != 3 || got[1] != "c" || got[2] != "d" {
 		t.Errorf("editList: %v", got)
 	}

docs/operations.md

@@ -164,9 +164,9 @@ start again. A disabled rule keeps its count for when it is enabled.
 resets; `hc rule reset-quota ID` starts one again (`--address` one device's,
 `--give 5gbytes` a top-up until the next reset). housecarld resets
 quotas at midnight when their period ends (late, if it was stopped then),
-and saves their usage every five minutes and when it stops; after a
-reboot it puts it back. A reboot without a clean stop loses at most the
-last five minutes' count.
+and saves their usage every five minutes, after every change and when
+it stops; after a reboot it puts it back. A reboot without a clean stop
+loses at most the last five minutes' count.
 
 ## The log
 

docs/rules.md

@@ -209,8 +209,8 @@ Statistics**.
 
 nftables counts a quota but never resets it, and forgets it when its
 table is replaced: Housecarl carries what each quota used across every
-change and restart, saves it every five minutes (and when housecarld
-stops) for after a reboot, and resets quotas when their period ends.
+change and restart, saves it every five minutes (and after each change,
+and when housecarld stops) for after a reboot, and resets quotas when their period ends.
 A per-address quota counts an address: a device that gets a new one
 starts again. With Steward's reservations IPv4 addresses stay; IPv6 is
 grouped by /64 (below).

internal/server/server.go

@@ -141,6 +141,16 @@ func New(cfg Config, eng *engine.Engine, fw Firewall, ifaces Interfaces, log *sl
 	s := &Server{cfg: cfg, eng: eng, fw: fw, ifaces: ifaces, store: eng.Store(), log: log, mux: http.NewServeMux(), limiter: newRateLimiter(),
 		web: web, started: time.Now(), logs: newLogRing(2000), nowFn: time.Now, unitLinks: nftablesUnitLinks}
 	s.routes()
+	// What the quotas used is saved after every change too, not only every
+	// five minutes: a change that counts a quota differently converts its
+	// usage, and a restart must not bring back the old kind.
+	prev := eng.OnChange
+	eng.OnChange = func(st *model.State) {
+		if prev != nil {
+			prev(st)
+		}
+		s.SaveQuotaUsage(context.Background())
+	}
 	return s
 }
 

internal/server/server_test.go

@@ -792,6 +792,21 @@ func TestQuotaResets(t *testing.T) {
 	}
 }
 
+// TestQuotaUsageSavedOnChange: a change saves what the quotas used at
+// once (a restart must not bring back usage counted the old way).
+func TestQuotaUsageSavedOnChange(t *testing.T) {
+	e := newEnv(t)
+	e.call(e.token, "POST", "/zones", model.Zone{Name: "lan", Interfaces: []string{"eth1"}}, nil)
+	e.fw.mu.Lock()
+	e.fw.quotas = nft.QuotaUsage{Quotas: map[string]int64{"quota_r7": 4242}}
+	e.fw.mu.Unlock()
+	e.call(e.token, "POST", "/zones", model.Zone{Name: "dmz", Interfaces: []string{"eth2"}}, nil)
+	v, err := e.eng.Store().GetMeta(context.Background(), QuotaUsageKey)
+	if err != nil || !strings.Contains(v, `"quota_r7":4242`) {
+		t.Errorf("saved usage %q %v", v, err)
+	}
+}
+
 // TestAutoConfirm: the client that made a change confirms it by itself:
 // with its token, from the same address, over a new connection. Nobody
 // else can, an older token can't, and with manual_confirm there is none.

internal/stats/collector.go

@@ -402,7 +402,10 @@ func (c *Collector) sample(ctx context.Context, minute int64) map[string]int64 {
 			for set, es := range t.QuotaSets {
 				kind := strings.TrimSuffix(strings.TrimSuffix(set, "_v4"), "_v6")
 				for _, e := range es {
-					addTop(kind, e.Addr, c.table.of(set+"/"+e.Addr, e.Used))
+					d := c.table.of(set+"/"+e.Addr, e.Used)
+					addTop(kind, e.Addr, d)
+					// A per-address quota's series is its addresses' sum.
+					add("quota:"+strings.TrimPrefix(kind, "quota_"), d)
 				}
 			}
 			for set, es := range t.DeviceSets {

internal/stats/collector_test.go

@@ -85,6 +85,10 @@ func TestCollector(t *testing.T) {
 	h.table.Counters["r1"] = nft.Counter{Packets: 10, Bytes: 1000}
 	h.table.Counters["r2"] = nft.Counter{Packets: 100, Bytes: 9000}
 	h.table.Quotas["quota_r5"] = 1 << 20
+	if h.table.QuotaSets == nil {
+		h.table.QuotaSets = map[string][]render.Element{}
+	}
+	h.table.QuotaSets["quota_r6_v4"] = []render.Element{{Addr: "10.0.10.5", Used: 1000}}
 	h.table.DeviceSets["stats_up_v4"] = []render.Element{{Addr: "10.0.10.5", Bytes: 5000}}
 
 	store, err := OpenStore(":memory:")
@@ -104,6 +108,7 @@ func TestCollector(t *testing.T) {
 	h.table.Counters["r1"] = nft.Counter{Packets: 25, Bytes: 2500}
 	h.table.Counters["p-input"] = nft.Counter{Packets: 3, Bytes: 180} // a new counter counts whole
 	h.table.Quotas["quota_r5"] = 3 << 20
+	h.table.QuotaSets["quota_r6_v4"] = []render.Element{{Addr: "10.0.10.5", Used: 2000}, {Addr: "10.0.10.6", Used: 500}} // a new entry counts whole
 	h.table.DeviceSets["stats_up_v4"] = []render.Element{{Addr: "10.0.10.5", Bytes: 8000}}
 	h.table.DeviceSets["stats_down_v6"] = []render.Element{{Addr: "2001:db8:1:2::", Bytes: 70000}}
 	for range 2 {
@@ -142,6 +147,7 @@ func TestCollector(t *testing.T) {
 	check("drop:forward", sum("drop:")["forward"], 15)
 	check("drop:input", sum("drop:")["input"], 3)
 	check("quota:r5", sum("quota:")["r5"], 2<<20)
+	check("quota:r6 (per address: the sum)", sum("quota:")["r6"], 1500)
 
 	s, err := q.Summary(ctx, time.Hour, 10)
 	if err != nil {

internal/web/static/assets/charts.js

@@ -331,7 +331,7 @@ export function heatmapTable(hm, unit) {
 // so the list is its own table view.
 // extra(key), when given, adds something after an entry's label (e.g. a
 // link that excludes it).
-export function barList(entries, { total, label = (k) => k, empty = "No data yet.", href, extra, format = compact } = {}) {
+export function barList(entries, { total, label = (k) => k, empty = "No data yet.", href, extra, format = compact, share: showShare = true } = {}) {
   if (!entries.length) return h("div", { class: "empty" }, empty);
   // Entries may be in a natural order (e.g. latency buckets), not by size.
   const peak = Math.max(1, ...entries.map((e) => e.count));
@@ -345,7 +345,7 @@ export function barList(entries, { total, label = (k) => k, empty = "No data yet
     return h("li", { title: (e.name ? e.name + " (" + name + ")" : name) + ": " + format(e.count) + " (" + share.toFixed(1) + "%)" },
       h("div", { class: "bar-text" },
         link ? h("a", { href: link, class: "bar-label" }, shown) : h("span", { class: "bar-label" }, shown),
-        h("span", { class: "bar-value" }, extra ? [extra(e.key), " "] : null, format(e.count), h("span", { class: "muted" }, " · " + share.toFixed(share < 10 ? 1 : 0) + "%"))),
+        h("span", { class: "bar-value" }, extra ? [extra(e.key), " "] : null, format(e.count), showShare ? h("span", { class: "muted" }, " · " + share.toFixed(share < 10 ? 1 : 0) + "%") : null)),
       h("div", { class: "bar-track" }, h("div", { class: "bar-fill", style: { width: Math.max(0.5, (e.count / peak) * 100) + "%" } })));
   }));
 }

internal/web/static/assets/pages/dashboard.js

@@ -3,7 +3,7 @@
 import { h, mount, table, badge, problemsList, time, plural } from "../ui.js";
 import { get, can } from "../api.js";
 import { onLeave } from "../app.js";
-import { describe, hits, humanCount, fmtBytes, usageBar, lookupNames, named } from "../widgets.js";
+import { describe, hits, humanCount, fmtBytes, usageBar, lookupNames, named, pct } from "../widgets.js";
 import { lineChart, chartCard, legend, timeTable, stepSeconds } from "../charts.js";
 import { bytes } from "./stats.js";
 
@@ -82,8 +82,8 @@ function quotaPanel(quotas, names) {
       { label: "Rule", render: (q) => h("a", { href: "#/rules?rule=" + q.rule_id }, q.rule) },
       { label: "Allowance", render: (q) => fmtBytes(q.quota.bytes) + (q.quota.per_source ? " each source" : q.quota.per_destination ? " each destination" : "") },
       { label: "Used", render: (q) => perAddress(q)
-        ? [plural((q.addresses || []).length, "address", "addresses"), q.addresses && q.addresses.length ? [", busiest ", named(names, q.addresses[0].address), " " + fmtBytes(used(q))] : ""]
-        : fmtBytes(q.used) },
+        ? [plural((q.addresses || []).length, "address", "addresses"), q.addresses && q.addresses.length ? [", busiest ", named(names, q.addresses[0].address), " " + fmtBytes(used(q)) + " (" + pct(used(q), q.quota.bytes) + ")"] : ""]
+        : fmtBytes(q.used) + " (" + pct(q.used, q.quota.bytes) + ")" },
       { label: "", render: (q) => usageBar(used(q), q.quota.bytes) },
       { label: "Next reset", render: (q) => q.next_reset ? time(q.next_reset) : h("span", { class: "muted" }, "by hand") },
     ], quotas));

internal/web/static/assets/pages/rules.js

@@ -2,7 +2,7 @@
 import { h, mount, table, modal, errorBox, field, input, select, checkbox, confirmDialog, badge, textarea, toast } from "../ui.js";
 import { get, api, change, can } from "../api.js";
 import { rerender } from "../app.js";
-import { names, matchEditor, describe, hits, counters, listEditor, rowClass, live, fmtBytes, quotas, usageBar, lookupNames, named } from "../widgets.js";
+import { names, matchEditor, describe, hits, counters, listEditor, rowClass, live, fmtBytes, quotas, usageBar, lookupNames, named, pct } from "../widgets.js";
 
 const chains = [
   ["input", "Input", "Traffic to the firewall itself."],
@@ -165,10 +165,10 @@ function quotaLine(r, q) {
   const each = qt.per_source ? " each source" : qt.per_destination ? " each destination" : "";
   const resets = qt.reset === "monthly" ? "monthly" : qt.reset || "reset by hand";
   let used = fmtBytes(qt.bytes) + each;
-  if (q && !each) used = fmtBytes(q.used) + " of " + fmtBytes(qt.bytes) + " used";
+  if (q && !each) used = fmtBytes(q.used) + " of " + fmtBytes(qt.bytes) + " used (" + pct(q.used, qt.bytes) + ")";
   else if (q) {
     const n = (q.addresses || []).length;
-    used += " · " + (n === 1 ? "1 address" : n + " addresses") + (n ? ", busiest " + fmtBytes(q.addresses[0].used) : "");
+    used += " · " + (n === 1 ? "1 address" : n + " addresses") + (n ? ", busiest " + fmtBytes(q.addresses[0].used) + " (" + pct(q.addresses[0].used, qt.bytes) + ")" : "");
   }
   const text = "quota " + used + " · " + resets;
   return h("div", { class: "muted small" }, q && r.pending !== "new"
@@ -193,12 +193,13 @@ async function quotaDialog(r, q) {
     perAddress ? table([
       { label: "Address", render: (a) => h("span", { class: "mono" }, named(names, a.address)) },
       { label: "Used", render: (a) => fmtBytes(a.used) },
+      { label: "", class: "num", tdClass: "num", render: (a) => pct(a.used, qt.bytes) },
       { label: "", render: (a) => usageBar(a.used, qt.bytes) },
       write ? { label: "", render: (a) => h("span", { class: "nowrap" },
         h("button", { class: "small", onclick: then(() => resetQuota(r, a.address)) }, "Reset"), " ",
         h("button", { class: "small", onclick: then(() => giveBack(r, a.address)) }, "Give back…")) } : null,
     ].filter(Boolean), q.addresses || [], { empty: "No address has used it since the last reset." })
-      : h("div", null, h("strong", null, fmtBytes(q.used)), " of " + fmtBytes(qt.bytes) + " used", usageBar(q.used, qt.bytes))),
+      : h("div", null, h("strong", null, fmtBytes(q.used)), " of " + fmtBytes(qt.bytes) + " used (" + pct(q.used, qt.bytes) + ")", usageBar(q.used, qt.bytes))),
   [
     ...(write ? [{ label: perAddress ? "Reset all" : "Reset", value: "reset" }, { label: perAddress ? "Give back to all…" : "Give back…", value: "give" }] : []),
     { label: "Close", value: null },

internal/web/static/assets/pages/stats.js

@@ -104,7 +104,7 @@ export async function render(el, params, query) {
     listCard("What they tried", "the logged drops' protocol and port", barList(s.blocked_ports, { empty: "No logged drops " + rangeLabel + "." })),
     listCard("Devices, received", "bytes from external zones, by inside address (IPv6 per /64)", s.devices_down.length ? barList(s.devices_down, { format: bytes, href: connHref }) : devHint),
     listCard("Devices, sent", "bytes to external zones", s.devices_up.length ? barList(s.devices_up, { format: bytes, href: connHref }) : devHint.cloneNode(true)),
-    d.quotas.length ? listCard("Quotas", "bytes counted against each rule's quota, " + rangeLabel, barList(d.quotas, { format: bytes, label: (k) => "rule #" + k.slice(1), href: () => "#/rules" })) : null,
+    d.quotas.length ? listCard("Quotas", "bytes counted against each rule's quota, " + rangeLabel, barList(d.quotas, { format: bytes, label: (k) => (d.hits.find((x) => x.key === k) || {}).name || "rule #" + k.slice(1), href: ruleHref, share: false })) : null,
     listCard("Rules that matched nothing", "in the last 30 days (or since statistics started): candidates to remove",
       unused.length ? h("div", null, h("ul", { class: "plain-list" }, unused.slice(0, 12).map((e) => h("li", null, h("a", { href: ruleHref(e.key) }, e.name || e.key)))),
         unused.length > 12 ? h("div", { class: "muted small" }, "and " + (unused.length - 12) + " more: hc stats rules --unused") : null) : h("div", { class: "empty" }, "Every rule matched something."))));

internal/web/static/assets/widgets.js

@@ -161,6 +161,13 @@ export function fmtBytes(n) {
   return n + " bytes";
 }
 
+// pct writes used as a share of total: "25%", "<1%" for a little.
+export function pct(used, total) {
+  if (!total || used <= 0) return "0%";
+  const p = used / total * 100;
+  return p < 1 ? "<1%" : Math.floor(p) + "%";
+}
+
 // usageBar draws used out of total as a bar, red once it is all used.
 export function usageBar(used, total) {
   const seg = h("div", { class: "seg leased" });

scripts/e2e.sh

@@ -322,7 +322,7 @@ send 2000000 >/dev/null 2>&1
 run "used up: the device is cut off" 1 send 1000
 run "the usage" 0 H quotas
 contains "e2e quota.*1.0 MB each.*by hand"
-contains "$CLIENT4 (e2e-client)  *1\.[0-9] MB"
+contains "$CLIENT4 (e2e-client)  *1\.[0-9] MB (1[0-9][0-9]%)"
 stop; start
 run "after a restart, still used up" 1 send 1000
 run "the usage after a restart" 0 H quotas
@@ -348,7 +348,7 @@ run "within the quota" 0 send 300000
 run "to one total" 0 H rule update "$QR" --quota 1mbytes,over
 not_contains "quota_restarts"
 run "the total carried over" 0 H quotas
-contains "e2e quota)  *[0-9.]* KB"
+contains "e2e quota)  *[0-9.]* KB ([0-9]*%)"
 run "per destination starts again" 0 H rule update "$QR" --quota 1mbytes,over,per-destination
 contains "quota_restarts"
 run "the quota rule goes" 0 H rule delete -y "$QR"