Add cluster membership and ACN transport

Introduce the first cluster runtime phase with a new ACN package (node keys, signed requests with replay protection, direct/tunnel/relay routing, and tunnel handling), membership manager (create/join/leave, heartbeats, sync, join tokens, admin APIs), and capability detection (including cross-platform disk usage). Wire cluster startup/shutdown into the core, mount `/cluster/acn/*` before session auth, reserve `cluster` from subservice paths, add a System Settings Cluster UI page, and document the architecture with new cluster READMEs and guidance updates.

CLAUDE.md

@@ -242,6 +242,38 @@ through AGI, while a **subservice** is an *external* program ArozOS launches,
 supervises and reverse-proxies. Use a webapp by default; reach for a subservice
 when the work doesn't fit the in-core JavaScript sandbox.
 
+## What the Cluster is
+
+Several independent ArozOS installations can form a **cluster** that exposes one
+logical computer (namespace, identity, compute) while each node keeps its own
+hardware, OS and storage and keeps working standalone when the cluster is away.
+The runtime lives in [`src/mod/cluster/`](src/mod/cluster/) and is documented in
+[`src/mod/cluster/README.md`](src/mod/cluster/README.md) — read that first.
+
+- **ACN** ([`src/mod/cluster/acn/`](src/mod/cluster/acn/)) is the node-to-node
+  protocol: Ed25519 node keys, signed requests with replay protection, and a
+  transport that routes **direct** (advertised URL, Cloudflare-friendly),
+  over a **tunnel** (a NAT-only node keeps one WebSocket open to a reachable
+  member) or by **relay** through the tunnel host. It is mounted at
+  `/cluster/acn/*` in [`src/main.router.go`](src/main.router.go) *before* the
+  user-session check; `cluster` is a reserved subservice path. Register new
+  signed endpoints with `acn.Server.HandleFunc` and call peers with
+  `acn.Transport.DoJSON` — never talk to node addresses directly.
+- **Membership** ([`src/mod/cluster/membership/`](src/mod/cluster/membership/))
+  is the cluster agent: create / join (pasteable join tokens) / leave, records
+  merged last-writer-wins by gossip, heartbeats, computed node states, health
+  and capability manifests ([`src/mod/cluster/capability/`](src/mod/cluster/capability/)).
+  Admin API `/system/cluster/*` and the System Settings page
+  [`src/web/SystemAO/cluster/cluster.html`](src/web/SystemAO/cluster/cluster.html);
+  wiring in [`src/cluster.go`](src/cluster.go).
+- **State** lives in its own key-value file `system/cluster.db` (never `ao.db`)
+  and the node key in `system/cluster/node.key`.
+- **Design rules:** whole files, never chunked storage; cross-node transfers in
+  ≤4 MB hash-verified chunks (Cloudflare limits); metadata consistency by
+  leader lease + replicated log (no Raft, 2-node clusters must work); jobs are
+  `.agi` scripts because nodes differ in architecture; keep everything portable
+  (build-tagged files for syscalls, see `capability/diskusage_*.go`).
+
 ## Build, run and test
 
 All Go commands run from `src/`:

src/cluster.go

@@ -2,7 +2,13 @@ package main
 
 import (
 	"net/http"
+	"path/filepath"
+	"sync"
+	"time"
 
+	"imuslab.com/arozos/mod/cluster/capability"
+	"imuslab.com/arozos/mod/cluster/membership"
+	"imuslab.com/arozos/mod/info/usageinfo"
 	"imuslab.com/arozos/mod/network/neighbour"
 	prout "imuslab.com/arozos/mod/prouter"
 )
@@ -12,16 +18,88 @@ import (
 	Author: tobychui
 
 	This is a section of the arozos core that handle cluster
-	related function endpoints
+	related function endpoints:
 
+	- Neighbourhood: mDNS discovery of nearby ArozOS hosts (LAN only)
+	- Cluster membership: the cluster agent of this node (mod/cluster/membership)
+	  reachable by other nodes under /cluster/acn/* (see main.router.go)
 */
 
 var (
 	NeighbourDiscoverer *neighbour.Discoverer
+	clusterManager      *membership.Manager
 )
 
+// clusterHealthProvider builds the load snapshot shipped with each heartbeat.
+// RAM figures are cached because reading them shells out on some platforms.
+type clusterHealthProvider struct {
+	mu        sync.Mutex
+	ramUsed   int64
+	ramTotal  int64
+	ramSample time.Time
+}
+
+func (p *clusterHealthProvider) snapshot() membership.Health {
+	h := membership.Health{}
+	if cpu, _, _, _, ready := usageinfo.GetCachedStats(); ready {
+		h.CPUUsage = cpu
+	}
+	p.mu.Lock()
+	if time.Since(p.ramSample) > time.Minute {
+		used, total := usageinfo.GetNumericRAMUsage()
+		if total > 0 {
+			p.ramUsed, p.ramTotal = used, total
+		}
+		p.ramSample = time.Now()
+	}
+	h.RAMUsed, h.RAMTotal = p.ramUsed, p.ramTotal
+	p.mu.Unlock()
+	if free, total, err := capability.DiskUsage(filepath.Clean(*root_directory)); err == nil {
+		h.DiskFree, h.DiskTotal = free, total
+	}
+	return h
+}
+
 func ClusterInit() {
-	//Only enable cluster scanning on mdns enabled mode
+	//Cluster agent: always available so a node can create or join a cluster
+	//from System Settings regardless of the LAN discovery features
+	health := &clusterHealthProvider{}
+	manager, err := membership.NewManager(membership.Option{
+		NodeID:      deviceUUID,
+		DBFile:      filepath.Join("system", "cluster.db"),
+		KeyFile:     filepath.Join("system", "cluster", "node.key"),
+		Version:     build_version + " " + internal_version,
+		DefaultName: *host_name,
+		Health:      health.snapshot,
+	})
+	if err != nil {
+		systemWideLogger.PrintAndLog("Cluster", "Unable to start cluster agent: "+err.Error(), err)
+	} else {
+		clusterManager = manager
+
+		adminRouter := prout.NewModuleRouter(prout.RouterOption{
+			ModuleName:  "System Setting",
+			AdminOnly:   true,
+			UserHandler: userHandler,
+			DeniedHandler: func(w http.ResponseWriter, r *http.Request) {
+				errorHandlePermissionDenied(w, r)
+			},
+		})
+		clusterManager.RegisterAdminRoutes(func(pattern string, handler func(http.ResponseWriter, *http.Request)) {
+			adminRouter.HandleFunc(pattern, handler)
+		})
+
+		registerSetting(settingModule{
+			Name:         "Cluster",
+			Desc:         "Create or join an ArozOS cluster",
+			IconPath:     "SystemAO/cluster/img/small_icon.png",
+			Group:        "Cluster",
+			StartDir:     "SystemAO/cluster/cluster.html",
+			RequireAdmin: true,
+		})
+	}
+
+	//Only enable neighbourhood scanning on mdns enabled mode
 	if *allow_mdns && MDNS != nil {
 		//Start the network discovery
 		thisDiscoverer := neighbour.NewDiscoverer(MDNS, sysdb)
@@ -56,13 +134,14 @@ func ClusterInit() {
 		router.HandleFunc("/system/cluster/scan", NeighbourDiscoverer.HandleScanningRequest)
 		router.HandleFunc("/system/cluster/record", NeighbourDiscoverer.HandleScanRecord)
 		router.HandleFunc("/system/cluster/wol", NeighbourDiscoverer.HandleWakeOnLan)
-
-		/*
-			Start and Cluster Server and Client
-		*/
-		//WIP
 	} else {
 		systemWideLogger.PrintAndLog("Cluster", "MDNS not enabled or startup failed. Skipping Cluster Scanner initiation.", nil)
 	}
+}
 
+// ClusterShutdown stops heartbeats and tunnels before the process exits.
+func ClusterShutdown() {
+	if clusterManager != nil {
+		clusterManager.Close()
+	}
 }

src/go.mod

@@ -117,7 +117,7 @@ require (
 	gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 // indirect
 	go4.org v0.0.0-20260112195520-a5071408f32f // indirect
 	golang.org/x/mod v0.37.0 // indirect
-	golang.org/x/sys v0.46.0 // indirect
+	golang.org/x/sys v0.46.0
 	golang.org/x/text v0.39.0 // indirect
 	golang.org/x/tools v0.47.0 // indirect
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect

src/main.go

@@ -38,6 +38,10 @@ func SetupCloseHandler() {
 }
 
 func executeShutdownSequence() {
+	//Shutdown cluster agent so peers stop routing to this node
+	systemWideLogger.PrintAndLog("System", "<!> Shutting down cluster agent", nil)
+	ClusterShutdown()
+
 	//Shutdown authAgent
 	systemWideLogger.PrintAndLog("System", "<!> Shutting down auth gateway", nil)
 	authAgent.Close()

src/main.router.go

@@ -64,6 +64,14 @@ func mrouter(h http.Handler) http.Handler {
 			}
 			h.ServeHTTP(w, r)
 
+		} else if len(r.URL.Path) >= len("/cluster/acn") && r.URL.Path[:12] == "/cluster/acn" {
+			//Cluster node-to-node protocol (ACN). Requests are authenticated by
+			//node signatures, not user sessions, see mod/cluster/acn
+			if clusterManager == nil {
+				errorHandleInternalServerError(w, r)
+				return
+			}
+			clusterManager.ACNHandler().ServeHTTP(w, r)
 		} else if len(r.URL.Path) >= len("/caldav") && r.URL.Path[:7] == "/caldav" {
 			//CalDAV sub-router (bidirectional calendar sync for iOS)
 			if CalDAVHandler == nil {

src/mod/cluster/README.md

@@ -0,0 +1,136 @@
+# ArozOS Cluster
+
+Several independent ArozOS installations expose **one logical computer
+interface** (namespace, identity, compute) while keeping their own hardware,
+OS and storage. Every node keeps working as a normal standalone ArozOS when
+the cluster is unreachable.
+
+This directory holds the cluster runtime. It is built in phases; the table at
+the end lists what exists today.
+
+## Packages
+
+| Package | Role |
+|---|---|
+| `acn/` | **ArozOS Cluster Node protocol** – the node-to-node transport. Ed25519 node keys, signed HTTP requests with replay protection, direct / tunnel / relay routing, and the WebSocket reverse tunnel for NAT-only nodes. |
+| `capability/` | Portable detection of what a node offers (OS, arch, cores, RAM, tools such as ffmpeg/docker/nvidia, CPU feature flags) plus `Requirements` matching for the scheduler, and a cross-platform `DiskUsage`. |
+| `membership/` | The cluster agent: create / join / leave, replicated membership records, join tokens, heartbeats, health, node states, tunnel host selection, and the admin (System Settings) endpoints. |
+| `wakeonlan/` | Wake-on-LAN packets for offline LAN neighbours. |
+
+Core wiring lives in [`src/cluster.go`](../../cluster.go); the ACN endpoints
+are mounted in [`src/main.router.go`](../../main.router.go) under
+`/cluster/acn/*` **before** the user-session check, because nodes authenticate
+with signatures rather than cookies. `cluster` is a reserved subservice path.
+
+Cluster state is stored in its own key-value database `system/cluster.db`,
+never in `ao.db`; the node key is `system/cluster/node.key`.
+
+## ACN – node-to-node protocol
+
+Every ACN request carries:
+
+```
+X-Aroz-Node:      <sender node UUID>
+X-Aroz-Cluster:   <cluster UUID>
+X-Aroz-Timestamp: <unix seconds>
+X-Aroz-Nonce:     <random hex>
+X-Aroz-Signature: base64(Ed25519(METHOD \n target \n cluster \n node \n ts \n nonce \n sha256(body)))
+```
+
+The receiver resolves the sender in its membership list, checks the cluster
+ID, a ±5 minute clock window, that the nonce is unseen, and the signature.
+Only then is the nonce spent, so junk requests cannot poison the replay cache.
+
+### Routing
+
+`acn.Transport.Do(nodeID, method, path, body)` picks the route; callers never
+deal with addresses:
+
+1. **Tunnel** – the peer has a live tunnel terminating on this node.
+2. **Direct** – the peer advertises a URL (behind Cloudflare is fine).
+3. **Relay** – the peer is tunnelled to another node: the request is sent to
+   `https://<via>/cluster/acn/relay/<peer>/cluster/acn/<path>` and that node
+   forwards it over its tunnel. The signature is made over the *original*
+   path, so both the relay and the final node verify the same signature.
+
+### Reverse tunnel
+
+A node without a public URL opens one WebSocket to a reachable member
+(`GET /cluster/acn/tunnel`, signed). Requests to it are multiplexed as binary
+frames `[kind][hdr len][header JSON][body]`, executed against the node's own
+ACN handler and answered on the same socket. Both ends ping every 30 s so
+Cloudflare's 100 s idle limit never trips; the client reconnects with backoff
+and re-picks a host (the admin's preferred one, otherwise the healthiest
+reachable member).
+
+Frames are capped at 16 MB, which keeps future file transfers at 4 MB chunks
+well under Cloudflare's 100 MB request limit.
+
+### Built-in endpoints
+
+| Endpoint | Auth | Purpose |
+|---|---|---|
+| `GET /cluster/acn/hello` | none | reachability probe (`{"acn":true,...}`) |
+| `POST /cluster/acn/ping` | signed | round-trip check |
+| `GET /cluster/acn/tunnel` | signed | tunnel upgrade |
+| `* /cluster/acn/relay/{node}/...` | signed | forward to a tunnelled node |
+| `POST /cluster/acn/join` | join token | add a node |
+| `POST /cluster/acn/heartbeat` | signed | liveness + membership exchange |
+| `GET /cluster/acn/members` | signed | full membership dump |
+| `POST /cluster/acn/members/sync` | signed | push membership changes |
+| `POST /cluster/acn/leave` | signed | sender leaves |
+| `POST /cluster/acn/evict` | signed | sender removed the receiver |
+
+## Membership
+
+Each node owns one `NodeRecord` (ID, name, public key, advertised URL, tunnel
+host, version, capabilities, admin state) with an `Updated` version stamp.
+Records travel with every heartbeat and are merged **last-writer-wins** per
+record, so membership converges without a master. A node is authoritative for
+its own record; other members may only change its admin state or remove it.
+Removed nodes stay as tombstones for 7 days so the removal propagates.
+
+Liveness is computed, not gossiped:
+
+| State | Meaning |
+|---|---|
+| `ONLINE` | heard from within 45 s |
+| `DEGRADED` | online but CPU/RAM ≥ 97 % or disk ≤ 2 % free |
+| `UNKNOWN` | silent for 45 s – 3 min |
+| `OFFLINE` | silent for more than 3 min |
+| `MAINTENANCE` / `DRAINING` | set by an admin, overrides the above |
+
+Heartbeats go to every peer every 15 s (full mesh; clusters are small). A
+NAT-only node is still seen as online by everyone because its tunnel host
+learns its `LastSeen` and gossips it.
+
+### Joining
+
+1. On a member **with an advertised URL**, generate a join token in
+   System Settings › Cluster. The token (`aroz-join:<base64 JSON>`) embeds the
+   cluster ID, the issuing node's URL and a secret whose SHA-256 is stored.
+2. Paste it on the new node. It POSTs its record to the issuer's `/join`,
+   receives the cluster info and member list, and starts heartbeating. If it
+   has no URL it immediately tunnels to the issuer.
+3. The issuer pushes the new member list to everyone else.
+
+## Admin API (`/system/cluster/*`, admin only)
+
+`status`, `create`, `join`, `leave`, `config`, `testurl`, `token/new`,
+`token/list`, `token/revoke`, `node/remove`, `node/state`, `node/probe`,
+`nodes`, `capabilities`. The UI is
+[`web/SystemAO/cluster/cluster.html`](../../web/SystemAO/cluster/cluster.html).
+
+## Roadmap
+
+| Phase | Status |
+|---|---|
+| 1 Membership (keys, ACN, tunnel/relay, join/leave, heartbeat, capabilities, health, settings UI) | done |
+| 2 Identity (forward-auth to an origin node, replicated accounts as fallback, signed user assertions) | planned |
+| 3 Metadata store (leader lease + replicated log, file records, locations, checksums) | planned |
+| 4 Unified namespace (`cluster:/` file system abstraction) | planned |
+| 5 Replication (whole files, 4 MB chunked transfer, checksum verified) | planned |
+| 6 AGI `cluster` library | planned |
+| 7 Job runtime (`run`, capability + locality aware scheduling) | planned |
+| 8 Map/Reduce | planned |
+| 9 Intelligent scheduling | planned |

src/mod/cluster/acn/acn_test.go

@@ -0,0 +1,498 @@
+package acn
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+/*
+	Test helpers
+*/
+
+type staticResolver struct {
+	mu    sync.RWMutex
+	peers map[string]*Peer
+}
+
+func newStaticResolver() *staticResolver {
+	return &staticResolver{peers: map[string]*Peer{}}
+}
+
+func (s *staticResolver) add(p *Peer) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.peers[p.ID] = p
+}
+
+func (s *staticResolver) ResolvePeer(id string) (*Peer, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	p, ok := s.peers[id]
+	return p, ok
+}
+
+type testNode struct {
+	id       string
+	key      *NodeKey
+	signer   *Signer
+	verifier *Verifier
+	hub      *TunnelHub
+	server   *Server
+	resolver *staticResolver
+	http     *httptest.Server
+	tr       *Transport
+}
+
+func newTestNode(t *testing.T, id string, cluster string, resolver *staticResolver) *testNode {
+	t.Helper()
+	key, err := GenerateNodeKey()
+	if err != nil {
+		t.Fatalf("GenerateNodeKey: %v", err)
+	}
+	n := &testNode{id: id, key: key, resolver: resolver}
+	n.signer = &Signer{NodeID: id, ClusterID: cluster, Key: key}
+	n.verifier = NewVerifier(func() string { return cluster }, resolver)
+	n.hub = NewTunnelHub(n.verifier)
+	n.server = NewServer(n.verifier, n.hub, "test")
+	n.http = httptest.NewServer(n.server)
+	n.tr = NewTransport(n.signer, resolver, n.hub, false)
+	t.Cleanup(func() {
+		n.hub.Close()
+		n.http.Close()
+	})
+	return n
+}
+
+func (n *testNode) peer(advertise bool, tunnelVia string) *Peer {
+	p := &Peer{ID: n.id, Name: n.id, PublicKey: n.key.Public, TunnelVia: tunnelVia}
+	if advertise {
+		p.AdvertiseURL = n.http.URL
+	}
+	return p
+}
+
+/*
+	Keys
+*/
+
+func TestNodeKeyRoundTrip(t *testing.T) {
+	key, err := GenerateNodeKey()
+	if err != nil {
+		t.Fatalf("GenerateNodeKey: %v", err)
+	}
+	parsed, err := ParseNodeKey(key.Encode())
+	if err != nil {
+		t.Fatalf("ParseNodeKey: %v", err)
+	}
+	if parsed.PublicKeyString() != key.PublicKeyString() {
+		t.Errorf("public key mismatch after round trip")
+	}
+	pub, err := DecodePublicKey(key.PublicKeyString())
+	if err != nil {
+		t.Fatalf("DecodePublicKey: %v", err)
+	}
+	if string(pub) != string(key.Public) {
+		t.Errorf("decoded public key differs")
+	}
+}
+
+func TestLoadOrCreateNodeKeyPersists(t *testing.T) {
+	dir := t.TempDir()
+	keyfile := filepath.Join(dir, "sub", "node.key")
+	first, err := LoadOrCreateNodeKey(keyfile)
+	if err != nil {
+		t.Fatalf("first load: %v", err)
+	}
+	second, err := LoadOrCreateNodeKey(keyfile)
+	if err != nil {
+		t.Fatalf("second load: %v", err)
+	}
+	if first.PublicKeyString() != second.PublicKeyString() {
+		t.Errorf("key not persisted: %s != %s", first.PublicKeyString(), second.PublicKeyString())
+	}
+}
+
+func TestParseNodeKeyInvalid(t *testing.T) {
+	cases := []string{"", "zz", "abcd"}
+	for _, c := range cases {
+		if _, err := ParseNodeKey(c); err == nil {
+			t.Errorf("ParseNodeKey(%q) expected error", c)
+		}
+	}
+	if _, err := DecodePublicKey("not-base64!"); err == nil {
+		t.Errorf("DecodePublicKey expected error")
+	}
+}
+
+/*
+	Signing
+*/
+
+func TestSignVerifyRoundTrip(t *testing.T) {
+	resolver := newStaticResolver()
+	a := newTestNode(t, "node-a", "cluster-1", resolver)
+	resolver.add(a.peer(true, ""))
+
+	body := []byte(`{"hello":"world"}`)
+	req := httptest.NewRequest(http.MethodPost, "/cluster/acn/ping?x=1", strings.NewReader(string(body)))
+	a.signer.Sign(req, SignedTarget("/cluster/acn/ping", "x=1"), body)
+
+	id, got, err := a.verifier.VerifyRequest(req)
+	if err != nil {
+		t.Fatalf("VerifyRequest: %v", err)
+	}
+	if id.NodeID != "node-a" {
+		t.Errorf("expected sender node-a, got %s", id.NodeID)
+	}
+	if string(got) != string(body) {
+		t.Errorf("body not preserved")
+	}
+}
+
+func TestVerifyRejects(t *testing.T) {
+	resolver := newStaticResolver()
+	a := newTestNode(t, "node-a", "cluster-1", resolver)
+	resolver.add(a.peer(true, ""))
+	stranger := newTestNode(t, "node-x", "cluster-1", resolver) //never added to resolver
+	other := newTestNode(t, "node-b", "cluster-2", resolver)
+	resolver.add(other.peer(true, ""))
+
+	body := []byte("payload")
+	mk := func() *http.Request {
+		return httptest.NewRequest(http.MethodPost, "/cluster/acn/ping", strings.NewReader(string(body)))
+	}
+
+	tests := []struct {
+		name string
+		req  func() *http.Request
+		want error
+	}{
+		{"unsigned", func() *http.Request { return mk() }, ErrNotSigned},
+		{"unknown node", func() *http.Request {
+			r := mk()
+			stranger.signer.Sign(r, "/cluster/acn/ping", body)
+			return r
+		}, ErrUnknownNode},
+		{"wrong cluster", func() *http.Request {
+			r := mk()
+			other.signer.Sign(r, "/cluster/acn/ping", body)
+			return r
+		}, ErrWrongCluster},
+		{"clock skew", func() *http.Request {
+			r := mk()
+			a.signer.SignAt(r, "/cluster/acn/ping", body, time.Now().Add(-MaxClockSkew-time.Minute))
+			return r
+		}, ErrClockSkew},
+		{"tampered body", func() *http.Request {
+			r := mk()
+			a.signer.Sign(r, "/cluster/acn/ping", []byte("different"))
+			return r
+		}, ErrBadSignature},
+		{"wrong path", func() *http.Request {
+			r := mk()
+			a.signer.Sign(r, "/cluster/acn/other", body)
+			return r
+		}, ErrBadSignature},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			_, _, err := a.verifier.VerifyRequest(tc.req())
+			if err != tc.want {
+				t.Errorf("expected %v, got %v", tc.want, err)
+			}
+		})
+	}
+}
+
+func TestVerifyRejectsReplay(t *testing.T) {
+	resolver := newStaticResolver()
+	a := newTestNode(t, "node-a", "cluster-1", resolver)
+	resolver.add(a.peer(true, ""))
+
+	body := []byte("once")
+	req := httptest.NewRequest(http.MethodPost, "/cluster/acn/ping", strings.NewReader(string(body)))
+	a.signer.Sign(req, "/cluster/acn/ping", body)
+	headers := req.Header.Clone()
+
+	if _, _, err := a.verifier.VerifyRequest(req); err != nil {
+		t.Fatalf("first verify: %v", err)
+	}
+	replay := httptest.NewRequest(http.MethodPost, "/cluster/acn/ping", strings.NewReader(string(body)))
+	replay.Header = headers
+	if _, _, err := a.verifier.VerifyRequest(replay); err != ErrReplay {
+		t.Errorf("expected ErrReplay, got %v", err)
+	}
+}
+
+func TestStripRelayPrefix(t *testing.T) {
+	tests := []struct {
+		path     string
+		node     string
+		original string
+		ok       bool
+	}{
+		{"/cluster/acn/relay/node-b/cluster/acn/ping", "node-b", "/cluster/acn/ping", true},
+		{"/cluster/acn/relay/node-b/cluster/acn/a/b", "node-b", "/cluster/acn/a/b", true},
+		{"/cluster/acn/ping", "", "/cluster/acn/ping", false},
+		{"/cluster/acn/relay/", "", "/cluster/acn/relay/", false},
+		{"/cluster/acn/relay/node-b", "", "/cluster/acn/relay/node-b", false},
+	}
+	for _, tc := range tests {
+		node, original, ok := StripRelayPrefix(tc.path)
+		if node != tc.node || original != tc.original || ok != tc.ok {
+			t.Errorf("StripRelayPrefix(%q) = (%q,%q,%v) want (%q,%q,%v)", tc.path, node, original, ok, tc.node, tc.original, tc.ok)
+		}
+	}
+}
+
+/*
+	Frames
+*/
+
+func TestFrameRoundTrip(t *testing.T) {
+	hdr := frameHeader{ID: "1", Method: "POST", Target: "/cluster/acn/ping?q=1", Headers: map[string][]string{"X-Test": {"a", "b"}}}
+	body := []byte("body-bytes")
+	data, err := encodeFrame(frameKindRequest, hdr, body)
+	if err != nil {
+		t.Fatalf("encodeFrame: %v", err)
+	}
+	kind, got, gotBody, err := decodeFrame(data)
+	if err != nil {
+		t.Fatalf("decodeFrame: %v", err)
+	}
+	if kind != frameKindRequest || got.ID != "1" || got.Target != hdr.Target || string(gotBody) != "body-bytes" {
+		t.Errorf("frame mismatch: %v %+v %q", kind, got, gotBody)
+	}
+	if len(got.Headers["X-Test"]) != 2 {
+		t.Errorf("headers not preserved")
+	}
+	if _, _, _, err := decodeFrame([]byte{1, 0}); err == nil {
+		t.Errorf("short frame should fail")
+	}
+	if _, err := encodeFrame(frameKindRequest, hdr, make([]byte, TunnelMaxFrame)); err != ErrFrameTooLarge {
+		t.Errorf("oversized frame should fail, got %v", err)
+	}
+}
+
+func TestWebSocketURL(t *testing.T) {
+	tests := []struct {
+		in   string
+		want string
+		err  bool
+	}{
+		{"https://node.example.com", "wss://node.example.com/cluster/acn/tunnel", false},
+		{"http://10.0.0.2:8080/", "ws://10.0.0.2:8080/cluster/acn/tunnel", false},
+		{"ftp://x", "", true},
+	}
+	for _, tc := range tests {
+		got, err := WebSocketURL(tc.in, TunnelPath)
+		if (err != nil) != tc.err {
+			t.Errorf("WebSocketURL(%q) err=%v", tc.in, err)
+			continue
+		}
+		if got != tc.want {
+			t.Errorf("WebSocketURL(%q)=%q want %q", tc.in, got, tc.want)
+		}
+	}
+}
+
+/*
+	Transport routes
+*/
+
+func TestTransportDirect(t *testing.T) {
+	resolver := newStaticResolver()
+	a := newTestNode(t, "node-a", "cluster-1", resolver)
+	b := newTestNode(t, "node-b", "cluster-1", resolver)
+	resolver.add(a.peer(true, ""))
+	resolver.add(b.peer(true, ""))
+
+	var out map[string]interface{}
+	err := a.tr.DoJSON(context.Background(), "node-b", http.MethodPost, BasePath+"/ping", map[string]string{"k": "v"}, &out)
+	if err != nil {
+		t.Fatalf("DoJSON: %v", err)
+	}
+	if out["node"] != "node-a" {
+		t.Errorf("ping should echo sender, got %v", out)
+	}
+
+	//Anonymous hello
+	resp, err := a.tr.DoURL(context.Background(), b.http.URL, http.MethodGet, BasePath+"/hello", nil, false)
+	if err != nil || resp.Status != 200 {
+		t.Fatalf("hello: %v %v", err, resp)
+	}
+	if !strings.Contains(string(resp.Body), `"acn":true`) {
+		t.Errorf("hello body unexpected: %s", resp.Body)
+	}
+}
+
+func TestTransportUnknownPeer(t *testing.T) {
+	resolver := newStaticResolver()
+	a := newTestNode(t, "node-a", "cluster-1", resolver)
+	if _, err := a.tr.Do(context.Background(), "ghost", http.MethodPost, BasePath+"/ping", nil); err != ErrUnknownNode {
+		t.Errorf("expected ErrUnknownNode, got %v", err)
+	}
+	resolver.add(&Peer{ID: "nat-only", PublicKey: a.key.Public})
+	if _, err := a.tr.Do(context.Background(), "nat-only", http.MethodPost, BasePath+"/ping", nil); err != ErrPeerNoEndpoint {
+		t.Errorf("expected ErrPeerNoEndpoint, got %v", err)
+	}
+}
+
+// connectTunnel attaches nat (no URL) to host and waits for the hub to see it.
+func connectTunnel(t *testing.T, nat *testNode, host *testNode) *TunnelClient {
+	t.Helper()
+	client := &TunnelClient{
+		Signer:  nat.signer,
+		Handler: nat.server,
+		PickHost: func() (TunnelHost, bool) {
+			return TunnelHost{NodeID: host.id, URL: host.http.URL}, true
+		},
+	}
+	client.Start()
+	t.Cleanup(client.Stop)
+	deadline := time.Now().Add(5 * time.Second)
+	for !host.hub.Connected(nat.id) {
+		if time.Now().After(deadline) {
+			t.Fatalf("tunnel from %s to %s never connected", nat.id, host.id)
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	return client
+}
+
+func TestTransportTunnelAndRelay(t *testing.T) {
+	resolver := newStaticResolver()
+	host := newTestNode(t, "node-host", "cluster-1", resolver)
+	nat := newTestNode(t, "node-nat", "cluster-1", resolver)
+	other := newTestNode(t, "node-other", "cluster-1", resolver)
+	resolver.add(host.peer(true, ""))
+	resolver.add(other.peer(true, ""))
+	resolver.add(nat.peer(false, host.id))
+
+	client := connectTunnel(t, nat, host)
+	if ok, h := client.Status(); !ok || h.NodeID != host.id {
+		t.Fatalf("client status wrong: %v %+v", ok, h)
+	}
+
+	//Host reaches nat over its own tunnel
+	var out map[string]interface{}
+	if err := host.tr.DoJSON(context.Background(), nat.id, http.MethodPost, BasePath+"/ping", map[string]int{"n": 1}, &out); err != nil {
+		t.Fatalf("host->nat via tunnel: %v", err)
+	}
+	if out["node"] != host.id {
+		t.Errorf("nat should see host as sender, got %v", out)
+	}
+
+	//Other reaches nat by relaying through host
+	out = nil
+	if err := other.tr.DoJSON(context.Background(), nat.id, http.MethodPost, BasePath+"/ping?via=relay", map[string]int{"n": 2}, &out); err != nil {
+		t.Fatalf("other->nat via relay: %v", err)
+	}
+	if out["node"] != other.id {
+		t.Errorf("nat should see other as original sender, got %v", out)
+	}
+
+	//Nat reaches other directly (other has a URL)
+	out = nil
+	if err := nat.tr.DoJSON(context.Background(), other.id, http.MethodPost, BasePath+"/ping", nil, &out); err != nil {
+		t.Fatalf("nat->other direct: %v", err)
+	}
+
+	//Relay refuses non-ACN paths
+	req, _ := http.NewRequest(http.MethodGet, host.http.URL+RelayPrefix+"/"+nat.id+"/system/secret", nil)
+	other.signer.Sign(req, "/system/secret", []byte{})
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("relay request: %v", err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Errorf("relay of non-ACN path should be rejected, got %d", resp.StatusCode)
+	}
+
+	//Drop the tunnel: host can no longer reach nat, relay returns bad gateway
+	client.Stop()
+	deadline := time.Now().Add(5 * time.Second)
+	for host.hub.Connected(nat.id) {
+		if time.Now().After(deadline) {
+			t.Fatalf("tunnel did not close")
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	if _, err := host.tr.Do(context.Background(), nat.id, http.MethodPost, BasePath+"/ping", nil); err != ErrTunnelNotConnected {
+		t.Errorf("expected ErrTunnelNotConnected, got %v", err)
+	}
+	r, err := other.tr.Do(context.Background(), nat.id, http.MethodPost, BasePath+"/ping", nil)
+	if err != nil {
+		t.Fatalf("relay after close: %v", err)
+	}
+	if r.Status != http.StatusBadGateway {
+		t.Errorf("relay after close expected 502, got %d", r.Status)
+	}
+}
+
+func TestTunnelRejectsUnsignedUpgrade(t *testing.T) {
+	resolver := newStaticResolver()
+	host := newTestNode(t, "node-host", "cluster-1", resolver)
+	resp, err := http.Get(host.http.URL + TunnelPath)
+	if err != nil {
+		t.Fatalf("GET tunnel: %v", err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusUnauthorized {
+		t.Errorf("unsigned tunnel upgrade expected 401, got %d", resp.StatusCode)
+	}
+}
+
+func TestTunnelClientReconnects(t *testing.T) {
+	resolver := newStaticResolver()
+	host := newTestNode(t, "node-host", "cluster-1", resolver)
+	nat := newTestNode(t, "node-nat", "cluster-1", resolver)
+	resolver.add(host.peer(true, ""))
+	resolver.add(nat.peer(false, host.id))
+
+	client := connectTunnel(t, nat, host)
+	client.Reconnect() //drops the socket, loop must dial again
+	deadline := time.Now().Add(10 * time.Second)
+	for {
+		//First wait for the drop to be observed on both ends
+		if ok, _ := client.Status(); !ok && !host.hub.Connected(nat.id) {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("tunnel did not drop after Reconnect")
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	for {
+		if ok, _ := client.Status(); ok && host.hub.Connected(nat.id) {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("tunnel did not reconnect")
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	if _, err := host.tr.Do(context.Background(), nat.id, http.MethodPost, BasePath+"/ping", nil); err != nil {
+		t.Errorf("ping after reconnect: %v", err)
+	}
+}
+
+func TestResponseError(t *testing.T) {
+	if (&Response{Status: 200}).Error() != nil {
+		t.Errorf("200 should not be an error")
+	}
+	if (&Response{Status: 500, Body: []byte("boom")}).Error() == nil {
+		t.Errorf("500 should be an error")
+	}
+	var nilResp *Response
+	if nilResp.Error() == nil {
+		t.Errorf("nil response should be an error")
+	}
+}

src/mod/cluster/acn/handlers.go

@@ -0,0 +1,151 @@
+package acn
+
+/*
+	ArozOS Cluster Node protocol (ACN) - server side plumbing
+
+	Server owns the HTTP mux mounted at BasePath on every node. Higher layers
+	(membership, metadata, storage, jobs) register their endpoints on it with
+	Handle / HandleFunc, which wrap the handler with signature verification.
+
+	Built-in endpoints:
+		GET  /cluster/acn/hello              anonymous reachability probe
+		POST /cluster/acn/ping               signed round trip check
+		GET  /cluster/acn/tunnel             tunnel upgrade for NAT-only nodes
+		*    /cluster/acn/relay/{node}/...   forward to a node tunnelled here
+*/
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// AuthenticatedHandler is a handler that only runs for verified members.
+type AuthenticatedHandler func(w http.ResponseWriter, r *http.Request, sender *SignedIdentity, body []byte)
+
+// HelloResponse is the anonymous reachability reply.
+type HelloResponse struct {
+	ACN       bool   `json:"acn"`
+	Version   string `json:"version"`
+	InCluster bool   `json:"inCluster"`
+	Time      int64  `json:"time"`
+}
+
+// Server mounts the ACN endpoints of this node.
+type Server struct {
+	Verifier *Verifier
+	Hub      *TunnelHub
+	Version  string
+	mux      *http.ServeMux
+}
+
+// NewServer builds the ACN mux with the built-in endpoints registered.
+func NewServer(verifier *Verifier, hub *TunnelHub, version string) *Server {
+	s := &Server{
+		Verifier: verifier,
+		Hub:      hub,
+		Version:  version,
+		mux:      http.NewServeMux(),
+	}
+	s.mux.HandleFunc(BasePath+"/hello", s.handleHello)
+	s.mux.HandleFunc(RelayPrefix+"/", s.handleRelay)
+	if hub != nil {
+		s.mux.HandleFunc(TunnelPath, hub.HandleTunnel)
+	}
+	s.HandleFunc(BasePath+"/ping", func(w http.ResponseWriter, r *http.Request, sender *SignedIdentity, body []byte) {
+		WriteJSON(w, map[string]interface{}{"pong": true, "node": sender.NodeID, "time": time.Now().Unix()})
+	})
+	return s
+}
+
+// ServeHTTP lets the server be mounted in the main router.
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.mux.ServeHTTP(w, r)
+}
+
+// HandleFunc registers a signed endpoint. Requests failing verification get 401.
+func (s *Server) HandleFunc(path string, handler AuthenticatedHandler) {
+	s.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
+		sender, body, err := s.Verifier.VerifyRequest(r)
+		if err != nil {
+			status := http.StatusUnauthorized
+			if errors.Is(err, ErrBodyTooLarge) {
+				status = http.StatusRequestEntityTooLarge
+			}
+			http.Error(w, err.Error(), status)
+			return
+		}
+		handler(w, r, sender, body)
+	})
+}
+
+// HandleRaw registers an endpoint without verification. Only for endpoints
+// that carry their own credentials (join tokens) or are intentionally public.
+func (s *Server) HandleRaw(path string, handler http.HandlerFunc) {
+	s.mux.HandleFunc(path, handler)
+}
+
+func (s *Server) handleHello(w http.ResponseWriter, r *http.Request) {
+	WriteJSON(w, HelloResponse{
+		ACN:       true,
+		Version:   s.Version,
+		InCluster: s.Verifier.ClusterID() != "",
+		Time:      time.Now().Unix(),
+	})
+}
+
+// handleRelay verifies the sender and forwards the request over the tunnel of
+// the target node. The signature (made over the original path) is preserved.
+func (s *Server) handleRelay(w http.ResponseWriter, r *http.Request) {
+	target, original, ok := StripRelayPrefix(r.URL.Path)
+	if !ok {
+		http.Error(w, "invalid relay path", http.StatusBadRequest)
+		return
+	}
+	if !strings.HasPrefix(original, BasePath+"/") {
+		http.Error(w, "relay only forwards ACN paths", http.StatusBadRequest)
+		return
+	}
+	_, body, err := s.Verifier.VerifyRequest(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusUnauthorized)
+		return
+	}
+	if s.Hub == nil || !s.Hub.Connected(target) {
+		http.Error(w, ErrTunnelNotConnected.Error(), http.StatusBadGateway)
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), DefaultRequestTimeout)
+	defer cancel()
+	resp, err := s.Hub.Do(ctx, target, r.Method, SignedTarget(original, r.URL.RawQuery), signedHeaders(r), body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadGateway)
+		return
+	}
+	for k, vals := range resp.Header {
+		for _, v := range vals {
+			w.Header().Add(k, v)
+		}
+	}
+	w.WriteHeader(resp.Status)
+	w.Write(resp.Body)
+}
+
+// WriteJSON encodes v as the JSON response body.
+func WriteJSON(w http.ResponseWriter, v interface{}) {
+	js, err := json.Marshal(v)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(js)
+}
+
+// WriteError sends a plain text error with the given status.
+func WriteError(w http.ResponseWriter, status int, msg string) {
+	http.Error(w, msg, status)
+}

src/mod/cluster/acn/keys.go

@@ -0,0 +1,106 @@
+package acn
+
+/*
+	ArozOS Cluster Node protocol (ACN) - node identity keys
+
+	Every ArozOS node that takes part in a cluster owns one Ed25519 key pair.
+	The public key is published in the node's membership record when it joins
+	a cluster; the private key never leaves the node and is used to sign every
+	node-to-node request (see signing.go).
+
+	The key is stored as a hex encoded 32 byte seed so it survives restarts.
+*/
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// NodeKey is the Ed25519 identity of this node.
+type NodeKey struct {
+	Private ed25519.PrivateKey
+	Public  ed25519.PublicKey
+}
+
+// GenerateNodeKey creates a brand new random node key pair.
+func GenerateNodeKey() (*NodeKey, error) {
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+	return &NodeKey{Private: priv, Public: pub}, nil
+}
+
+// LoadOrCreateNodeKey loads the node key stored at keyfile, creating and
+// persisting a new one when the file does not exist yet.
+func LoadOrCreateNodeKey(keyfile string) (*NodeKey, error) {
+	content, err := os.ReadFile(keyfile)
+	if err == nil {
+		return ParseNodeKey(strings.TrimSpace(string(content)))
+	}
+	if !errors.Is(err, os.ErrNotExist) {
+		return nil, err
+	}
+
+	key, err := GenerateNodeKey()
+	if err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(filepath.Dir(keyfile), 0700); err != nil {
+		return nil, err
+	}
+	if err := os.WriteFile(keyfile, []byte(key.Encode()), 0600); err != nil {
+		return nil, err
+	}
+	return key, nil
+}
+
+// ParseNodeKey rebuilds a key pair from the hex encoded seed produced by Encode.
+func ParseNodeKey(hexSeed string) (*NodeKey, error) {
+	seed, err := hex.DecodeString(strings.TrimSpace(hexSeed))
+	if err != nil {
+		return nil, errors.New("invalid node key encoding: " + err.Error())
+	}
+	if len(seed) != ed25519.SeedSize {
+		return nil, errors.New("invalid node key length")
+	}
+	priv := ed25519.NewKeyFromSeed(seed)
+	return &NodeKey{
+		Private: priv,
+		Public:  priv.Public().(ed25519.PublicKey),
+	}, nil
+}
+
+// Encode returns the hex encoded seed of the private key for persistence.
+func (k *NodeKey) Encode() string {
+	return hex.EncodeToString(k.Private.Seed())
+}
+
+// PublicKeyString returns the base64 encoded public key that is published to
+// other cluster members.
+func (k *NodeKey) PublicKeyString() string {
+	return EncodePublicKey(k.Public)
+}
+
+// EncodePublicKey converts a public key into its transport encoding.
+func EncodePublicKey(pub ed25519.PublicKey) string {
+	return base64.StdEncoding.EncodeToString(pub)
+}
+
+// DecodePublicKey parses the base64 public key of a peer.
+func DecodePublicKey(encoded string) (ed25519.PublicKey, error) {
+	raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(encoded))
+	if err != nil {
+		return nil, errors.New("invalid public key encoding: " + err.Error())
+	}
+	if len(raw) != ed25519.PublicKeySize {
+		return nil, errors.New("invalid public key length")
+	}
+	return ed25519.PublicKey(raw), nil
+}

src/mod/cluster/acn/signing.go

@@ -0,0 +1,307 @@
+package acn
+
+/*
+	ArozOS Cluster Node protocol (ACN) - request signing
+
+	Node-to-node requests do not use user sessions. Every request carries a
+	set of headers that identify the sending node and an Ed25519 signature
+	over the request method, target, cluster, node, timestamp, nonce and body
+	hash. Receivers look the sender up in the membership list (through a
+	PeerResolver), check the clock skew, reject replayed nonces and verify the
+	signature with the sender's published public key.
+
+	Relayed requests (see RelayPrefix) are signed over the target path with the
+	relay prefix removed, so both the relaying node and the final node can
+	verify the very same signature.
+*/
+
+import (
+	"bytes"
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	HeaderNode      = "X-Aroz-Node"
+	HeaderCluster   = "X-Aroz-Cluster"
+	HeaderTimestamp = "X-Aroz-Timestamp"
+	HeaderNonce     = "X-Aroz-Nonce"
+	HeaderSignature = "X-Aroz-Signature"
+
+	// BasePath is the URL prefix every ACN endpoint lives under on each node.
+	BasePath = "/cluster/acn"
+	// RelayPrefix is the prefix used to ask a node to forward a request to a
+	// node that is only reachable through a tunnel terminating at that node.
+	// The full form is RelayPrefix + "/" + targetNodeID + originalPath.
+	RelayPrefix = BasePath + "/relay"
+
+	// MaxClockSkew is the accepted difference between sender and receiver clocks.
+	MaxClockSkew = 5 * time.Minute
+	// MaxSignedBody limits how much of a request body a verifier will buffer.
+	MaxSignedBody = 64 << 20
+)
+
+var (
+	ErrNotSigned      = errors.New("request is not signed")
+	ErrUnknownNode    = errors.New("sender is not a member of this cluster")
+	ErrWrongCluster   = errors.New("request targets another cluster")
+	ErrClockSkew      = errors.New("request timestamp outside the accepted window")
+	ErrReplay         = errors.New("request nonce already seen")
+	ErrBadSignature   = errors.New("request signature invalid")
+	ErrBodyTooLarge   = errors.New("request body exceeds signed body limit")
+	ErrPeerNoEndpoint = errors.New("peer has no reachable endpoint")
+)
+
+// Peer is the transport-level view of a cluster member.
+type Peer struct {
+	ID           string
+	Name         string
+	PublicKey    ed25519.PublicKey
+	AdvertiseURL string //Base URL where the peer's HTTP server can be reached, empty for NAT-only nodes
+	TunnelVia    string //Node ID that terminates the peer's tunnel when it has no AdvertiseURL
+}
+
+// PeerResolver is implemented by the membership layer so the transport can
+// look up public keys and endpoints without importing it.
+type PeerResolver interface {
+	ResolvePeer(nodeID string) (*Peer, bool)
+}
+
+// Signer produces the ACN headers for outgoing requests.
+type Signer struct {
+	NodeID    string
+	ClusterID string
+	Key       *NodeKey
+}
+
+// SignedIdentity is what a verifier learns about a valid request.
+type SignedIdentity struct {
+	NodeID    string
+	ClusterID string
+	Peer      *Peer
+}
+
+// canonicalString builds the byte string that is signed and verified.
+func canonicalString(method, target, clusterID, nodeID, timestamp, nonce string, bodyHash []byte) []byte {
+	var b bytes.Buffer
+	b.WriteString(strings.ToUpper(method))
+	b.WriteByte('\n')
+	b.WriteString(target)
+	b.WriteByte('\n')
+	b.WriteString(clusterID)
+	b.WriteByte('\n')
+	b.WriteString(nodeID)
+	b.WriteByte('\n')
+	b.WriteString(timestamp)
+	b.WriteByte('\n')
+	b.WriteString(nonce)
+	b.WriteByte('\n')
+	b.WriteString(hex.EncodeToString(bodyHash))
+	return b.Bytes()
+}
+
+// SignedTarget returns the path (plus query when present) a request is signed over.
+func SignedTarget(path string, rawQuery string) string {
+	if rawQuery == "" {
+		return path
+	}
+	return path + "?" + rawQuery
+}
+
+// StripRelayPrefix removes the relay routing prefix from a request path and
+// returns the target node ID and the original path. ok is false when the path
+// is not a relay path.
+func StripRelayPrefix(path string) (targetNode string, originalPath string, ok bool) {
+	if !strings.HasPrefix(path, RelayPrefix+"/") {
+		return "", path, false
+	}
+	rest := strings.TrimPrefix(path, RelayPrefix+"/")
+	idx := strings.Index(rest, "/")
+	if idx <= 0 {
+		return "", path, false
+	}
+	return rest[:idx], rest[idx:], true
+}
+
+func newNonce() string {
+	buf := make([]byte, 16)
+	rand.Read(buf)
+	return hex.EncodeToString(buf)
+}
+
+// Sign adds the ACN authentication headers to req. target must be the path
+// (and query) the receiver will verify against, which for relayed requests is
+// the path without the relay prefix. body is the full request body.
+func (s *Signer) Sign(req *http.Request, target string, body []byte) {
+	s.SignAt(req, target, body, time.Now())
+}
+
+// SignAt is Sign with an explicit timestamp, used by tests.
+func (s *Signer) SignAt(req *http.Request, target string, body []byte, at time.Time) {
+	ts := strconv.FormatInt(at.Unix(), 10)
+	nonce := newNonce()
+	sum := sha256.Sum256(body)
+	msg := canonicalString(req.Method, target, s.ClusterID, s.NodeID, ts, nonce, sum[:])
+	sig := ed25519.Sign(s.Key.Private, msg)
+
+	req.Header.Set(HeaderNode, s.NodeID)
+	req.Header.Set(HeaderCluster, s.ClusterID)
+	req.Header.Set(HeaderTimestamp, ts)
+	req.Header.Set(HeaderNonce, nonce)
+	req.Header.Set(HeaderSignature, base64.StdEncoding.EncodeToString(sig))
+}
+
+// nonceCache remembers recently seen nonces for replay protection.
+type nonceCache struct {
+	mu      sync.Mutex
+	seen    map[string]int64 //nonce -> expiry unix time
+	lastGC  int64
+	ttl     int64
+	maxSize int
+}
+
+func newNonceCache(ttl time.Duration) *nonceCache {
+	return &nonceCache{
+		seen:    map[string]int64{},
+		ttl:     int64(ttl.Seconds()),
+		maxSize: 100000,
+	}
+}
+
+// remember returns false when the nonce was already recorded.
+func (c *nonceCache) remember(nonce string, now int64) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if now-c.lastGC > 60 || len(c.seen) > c.maxSize {
+		for k, exp := range c.seen {
+			if exp < now {
+				delete(c.seen, k)
+			}
+		}
+		c.lastGC = now
+	}
+	if _, exists := c.seen[nonce]; exists {
+		return false
+	}
+	c.seen[nonce] = now + c.ttl
+	return true
+}
+
+// Verifier checks incoming ACN requests.
+type Verifier struct {
+	ClusterID func() string //Returns the cluster this node belongs to, empty when not in a cluster
+	Resolver  PeerResolver
+	nonces    *nonceCache
+	Now       func() time.Time
+}
+
+// NewVerifier creates a verifier backed by the given resolver.
+func NewVerifier(clusterID func() string, resolver PeerResolver) *Verifier {
+	return &Verifier{
+		ClusterID: clusterID,
+		Resolver:  resolver,
+		nonces:    newNonceCache(MaxClockSkew * 2),
+		Now:       time.Now,
+	}
+}
+
+// ReadBody buffers and restores a request body so it can be hashed and then
+// consumed by the handler.
+func ReadBody(r *http.Request) ([]byte, error) {
+	if r.Body == nil {
+		return []byte{}, nil
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, MaxSignedBody+1))
+	if err != nil {
+		return nil, err
+	}
+	if len(body) > MaxSignedBody {
+		return nil, ErrBodyTooLarge
+	}
+	r.Body.Close()
+	r.Body = io.NopCloser(bytes.NewReader(body))
+	return body, nil
+}
+
+// Verify validates the ACN headers of r against target (the signed path) and
+// body and returns the identity of the sender. The request body is left
+// readable for the handler.
+func (v *Verifier) Verify(r *http.Request, target string, body []byte) (*SignedIdentity, error) {
+	nodeID := r.Header.Get(HeaderNode)
+	clusterID := r.Header.Get(HeaderCluster)
+	ts := r.Header.Get(HeaderTimestamp)
+	nonce := r.Header.Get(HeaderNonce)
+	sigEnc := r.Header.Get(HeaderSignature)
+	if nodeID == "" || clusterID == "" || ts == "" || nonce == "" || sigEnc == "" {
+		return nil, ErrNotSigned
+	}
+
+	localCluster := v.ClusterID()
+	if localCluster == "" || localCluster != clusterID {
+		return nil, ErrWrongCluster
+	}
+
+	tsInt, err := strconv.ParseInt(ts, 10, 64)
+	if err != nil {
+		return nil, ErrClockSkew
+	}
+	now := v.Now().Unix()
+	skew := now - tsInt
+	if skew < 0 {
+		skew = -skew
+	}
+	if skew > int64(MaxClockSkew.Seconds()) {
+		return nil, ErrClockSkew
+	}
+
+	peer, ok := v.Resolver.ResolvePeer(nodeID)
+	if !ok || peer == nil || len(peer.PublicKey) != ed25519.PublicKeySize {
+		return nil, ErrUnknownNode
+	}
+
+	sig, err := base64.StdEncoding.DecodeString(sigEnc)
+	if err != nil {
+		return nil, ErrBadSignature
+	}
+	sum := sha256.Sum256(body)
+	msg := canonicalString(r.Method, target, clusterID, nodeID, ts, nonce, sum[:])
+	if !ed25519.Verify(peer.PublicKey, msg, sig) {
+		return nil, ErrBadSignature
+	}
+
+	//Signature is valid, only now spend the nonce so bad requests cannot poison the cache
+	if !v.nonces.remember(nodeID+":"+nonce, now) {
+		return nil, ErrReplay
+	}
+
+	return &SignedIdentity{NodeID: nodeID, ClusterID: clusterID, Peer: peer}, nil
+}
+
+// VerifyRequest is the common entry point for HTTP handlers: it buffers the
+// body, derives the signed target (stripping the relay prefix when present)
+// and verifies the request.
+func (v *Verifier) VerifyRequest(r *http.Request) (*SignedIdentity, []byte, error) {
+	body, err := ReadBody(r)
+	if err != nil {
+		return nil, nil, err
+	}
+	path := r.URL.Path
+	if _, original, ok := StripRelayPrefix(path); ok {
+		path = original
+	}
+	identity, err := v.Verify(r, SignedTarget(path, r.URL.RawQuery), body)
+	if err != nil {
+		return nil, nil, err
+	}
+	return identity, body, nil
+}

src/mod/cluster/acn/transport.go

@@ -0,0 +1,204 @@
+package acn
+
+/*
+	ArozOS Cluster Node protocol (ACN) - outbound transport
+
+	Transport.Do sends one signed request to a peer, picking the route:
+
+		1. the peer is tunnelled to THIS node          -> over the tunnel
+		2. the peer advertises a URL                   -> direct HTTPS
+		3. the peer is tunnelled to another node       -> HTTPS to that node's
+		                                                   relay endpoint, which
+		                                                   forwards over its tunnel
+
+	Callers never deal with routing; higher layers just name the node.
+*/
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// DefaultRequestTimeout bounds one node-to-node request. Cloudflare returns
+// 524 after 100 seconds, so anything long-running must be split into chunks.
+const DefaultRequestTimeout = 60 * time.Second
+
+// Transport sends signed requests to cluster peers.
+type Transport struct {
+	Signer   *Signer
+	Resolver PeerResolver
+	Hub      *TunnelHub //Tunnels terminating at this node, may be nil
+	Client   *http.Client
+}
+
+func tlsConfig(insecure bool) *tls.Config {
+	if !insecure {
+		return nil
+	}
+	return &tls.Config{InsecureSkipVerify: true} //Admin opted in for self-signed LAN certificates
+}
+
+// NewTransport creates a transport. insecureTLS accepts self-signed peer
+// certificates, which the admin can enable for LAN-only clusters.
+func NewTransport(signer *Signer, resolver PeerResolver, hub *TunnelHub, insecureTLS bool) *Transport {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.TLSClientConfig = tlsConfig(insecureTLS)
+	tr.MaxIdleConnsPerHost = 8
+	return &Transport{
+		Signer:   signer,
+		Resolver: resolver,
+		Hub:      hub,
+		Client: &http.Client{
+			Transport: tr,
+			Timeout:   DefaultRequestTimeout,
+		},
+	}
+}
+
+// SetInsecureTLS toggles acceptance of self-signed peer certificates.
+func (t *Transport) SetInsecureTLS(insecure bool) {
+	if tr, ok := t.Client.Transport.(*http.Transport); ok {
+		tr.TLSClientConfig = tlsConfig(insecure)
+		tr.CloseIdleConnections()
+	}
+}
+
+// JoinURL glues a node base URL and an ACN path.
+func JoinURL(baseURL string, path string) string {
+	return strings.TrimRight(baseURL, "/") + path
+}
+
+// signedHeaders returns only the headers a relayed or tunnelled request must carry.
+func signedHeaders(req *http.Request) http.Header {
+	h := http.Header{}
+	for _, k := range []string{HeaderNode, HeaderCluster, HeaderTimestamp, HeaderNonce, HeaderSignature, "Content-Type"} {
+		if v := req.Header.Get(k); v != "" {
+			h.Set(k, v)
+		}
+	}
+	return h
+}
+
+func readResponse(resp *http.Response) (*Response, error) {
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, MaxSignedBody))
+	if err != nil {
+		return nil, err
+	}
+	return &Response{Status: resp.StatusCode, Header: resp.Header, Body: body}, nil
+}
+
+// DoURL sends a request straight to a base URL. When sign is true the ACN
+// headers are added; otherwise the request is anonymous (used before a node
+// has joined, e.g. for hello and join).
+func (t *Transport) DoURL(ctx context.Context, baseURL string, method string, path string, body []byte, sign bool) (*Response, error) {
+	if body == nil {
+		body = []byte{}
+	}
+	req, err := http.NewRequestWithContext(ctx, method, JoinURL(baseURL, path), bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	if len(body) > 0 {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	if sign {
+		if t.Signer == nil {
+			return nil, errors.New("transport has no signer")
+		}
+		t.Signer.Sign(req, path, body)
+	}
+	resp, err := t.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	return readResponse(resp)
+}
+
+// Do sends a signed request to the named cluster node. path is the ACN path
+// including any query string.
+func (t *Transport) Do(ctx context.Context, nodeID string, method string, path string, body []byte) (*Response, error) {
+	if body == nil {
+		body = []byte{}
+	}
+	if t.Signer == nil {
+		return nil, errors.New("transport has no signer")
+	}
+
+	//Route 1: tunnelled to this node
+	if t.Hub != nil && t.Hub.Connected(nodeID) {
+		req, _ := http.NewRequest(method, path, nil)
+		if len(body) > 0 {
+			req.Header.Set("Content-Type", "application/json")
+		}
+		t.Signer.Sign(req, path, body)
+		return t.Hub.Do(ctx, nodeID, method, path, signedHeaders(req), body)
+	}
+
+	peer, ok := t.Resolver.ResolvePeer(nodeID)
+	if !ok || peer == nil {
+		return nil, ErrUnknownNode
+	}
+
+	//Route 2: direct
+	if peer.AdvertiseURL != "" {
+		return t.DoURL(ctx, peer.AdvertiseURL, method, path, body, true)
+	}
+
+	//Route 3: relay through the node terminating the peer's tunnel
+	if peer.TunnelVia != "" && peer.TunnelVia != t.Signer.NodeID {
+		via, ok := t.Resolver.ResolvePeer(peer.TunnelVia)
+		if !ok || via == nil || via.AdvertiseURL == "" {
+			return nil, ErrPeerNoEndpoint
+		}
+		req, err := http.NewRequestWithContext(ctx, method, JoinURL(via.AdvertiseURL, RelayPrefix+"/"+nodeID+path), bytes.NewReader(body))
+		if err != nil {
+			return nil, err
+		}
+		if len(body) > 0 {
+			req.Header.Set("Content-Type", "application/json")
+		}
+		t.Signer.Sign(req, path, body) //signed over the ORIGINAL path
+		resp, err := t.Client.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		return readResponse(resp)
+	}
+
+	if peer.TunnelVia == t.Signer.NodeID {
+		return nil, ErrTunnelNotConnected
+	}
+	return nil, ErrPeerNoEndpoint
+}
+
+// DoJSON posts in as JSON to a node and decodes the JSON reply into out
+// (which may be nil). Non-2xx replies become errors.
+func (t *Transport) DoJSON(ctx context.Context, nodeID string, method string, path string, in interface{}, out interface{}) error {
+	var body []byte
+	if in != nil {
+		js, err := json.Marshal(in)
+		if err != nil {
+			return err
+		}
+		body = js
+	}
+	resp, err := t.Do(ctx, nodeID, method, path, body)
+	if err != nil {
+		return err
+	}
+	if err := resp.Error(); err != nil {
+		return err
+	}
+	if out != nil && len(resp.Body) > 0 {
+		return json.Unmarshal(resp.Body, out)
+	}
+	return nil
+}

src/mod/cluster/acn/tunnel.go

@@ -0,0 +1,640 @@
+package acn
+
+/*
+	ArozOS Cluster Node protocol (ACN) - reverse tunnel
+
+	Nodes that sit behind NAT or otherwise have no public URL keep one
+	persistent WebSocket connection open to a reachable peer (the "host").
+	HTTP requests destined for the tunnelled node are multiplexed over that
+	socket as binary frames, executed against the tunnelled node's local ACN
+	handler, and the responses are sent back over the same socket.
+
+	Frame layout (binary WebSocket message):
+
+		[1 byte kind][4 byte big-endian header length][header JSON][body bytes]
+
+	Requests are signed by the ORIGINAL sender, so the tunnelled node verifies
+	the same signature the host (or a relaying node) already verified.
+
+	Cloudflare terminates idle WebSockets after roughly 100 seconds, so both
+	ends ping every TunnelPingInterval.
+*/
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gorilla/websocket"
+	"imuslab.com/arozos/mod/info/logger"
+)
+
+const (
+	frameKindRequest  byte = 1
+	frameKindResponse byte = 2
+
+	TunnelPath         = BasePath + "/tunnel"
+	TunnelPingInterval = 30 * time.Second
+	TunnelReadTimeout  = 90 * time.Second
+	// TunnelMaxFrame bounds a single multiplexed request or response so a 4MB
+	// transfer chunk plus headers always fits with room to spare.
+	TunnelMaxFrame = 16 << 20
+)
+
+var (
+	ErrTunnelNotConnected = errors.New("target node is not connected through a tunnel on this node")
+	ErrTunnelClosed       = errors.New("tunnel closed")
+	ErrFrameTooLarge      = errors.New("tunnel frame exceeds maximum size")
+)
+
+type frameHeader struct {
+	ID      string              `json:"id"`
+	Method  string              `json:"m,omitempty"`
+	Target  string              `json:"t,omitempty"` //path plus query
+	Status  int                 `json:"s,omitempty"`
+	Headers map[string][]string `json:"h,omitempty"`
+}
+
+func encodeFrame(kind byte, hdr frameHeader, body []byte) ([]byte, error) {
+	hj, err := json.Marshal(hdr)
+	if err != nil {
+		return nil, err
+	}
+	if len(hj)+len(body)+5 > TunnelMaxFrame {
+		return nil, ErrFrameTooLarge
+	}
+	buf := make([]byte, 5, 5+len(hj)+len(body))
+	buf[0] = kind
+	binary.BigEndian.PutUint32(buf[1:5], uint32(len(hj)))
+	buf = append(buf, hj...)
+	buf = append(buf, body...)
+	return buf, nil
+}
+
+func decodeFrame(data []byte) (byte, frameHeader, []byte, error) {
+	var hdr frameHeader
+	if len(data) < 5 {
+		return 0, hdr, nil, errors.New("tunnel frame too short")
+	}
+	kind := data[0]
+	hlen := int(binary.BigEndian.Uint32(data[1:5]))
+	if hlen < 0 || 5+hlen > len(data) {
+		return 0, hdr, nil, errors.New("tunnel frame header length invalid")
+	}
+	if err := json.Unmarshal(data[5:5+hlen], &hdr); err != nil {
+		return 0, hdr, nil, err
+	}
+	return kind, hdr, data[5+hlen:], nil
+}
+
+// Response is the buffered result of a node-to-node request.
+type Response struct {
+	Status int
+	Header http.Header
+	Body   []byte
+}
+
+// Error turns a non-2xx response into a descriptive error, nil otherwise.
+func (r *Response) Error() error {
+	if r == nil {
+		return errors.New("empty response")
+	}
+	if r.Status >= 200 && r.Status < 300 {
+		return nil
+	}
+	msg := strings.TrimSpace(string(r.Body))
+	if len(msg) > 200 {
+		msg = msg[:200]
+	}
+	return errors.New("remote node returned " + http.StatusText(r.Status) + ": " + msg)
+}
+
+// tunnelConn is one live WebSocket shared by both ends of a tunnel.
+type tunnelConn struct {
+	ws      *websocket.Conn
+	writeMu sync.Mutex
+	pending map[string]chan *Response
+	pmu     sync.Mutex
+	closed  chan struct{}
+	once    sync.Once
+	seq     uint64
+}
+
+func newTunnelConn(ws *websocket.Conn) *tunnelConn {
+	ws.SetReadLimit(TunnelMaxFrame)
+	return &tunnelConn{
+		ws:      ws,
+		pending: map[string]chan *Response{},
+		closed:  make(chan struct{}),
+	}
+}
+
+func (c *tunnelConn) write(kind byte, hdr frameHeader, body []byte) error {
+	frame, err := encodeFrame(kind, hdr, body)
+	if err != nil {
+		return err
+	}
+	c.writeMu.Lock()
+	defer c.writeMu.Unlock()
+	select {
+	case <-c.closed:
+		return ErrTunnelClosed
+	default:
+	}
+	c.ws.SetWriteDeadline(time.Now().Add(60 * time.Second))
+	return c.ws.WriteMessage(websocket.BinaryMessage, frame)
+}
+
+func (c *tunnelConn) ping() error {
+	c.writeMu.Lock()
+	defer c.writeMu.Unlock()
+	return c.ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(10*time.Second))
+}
+
+func (c *tunnelConn) close() {
+	c.once.Do(func() {
+		close(c.closed)
+		c.ws.Close()
+		c.pmu.Lock()
+		for id, ch := range c.pending {
+			close(ch)
+			delete(c.pending, id)
+		}
+		c.pmu.Unlock()
+	})
+}
+
+func (c *tunnelConn) isClosed() bool {
+	select {
+	case <-c.closed:
+		return true
+	default:
+		return false
+	}
+}
+
+// roundTrip sends a request frame and waits for its response frame.
+func (c *tunnelConn) roundTrip(ctx context.Context, method string, target string, headers http.Header, body []byte) (*Response, error) {
+	c.pmu.Lock()
+	c.seq++
+	id := newNonce()
+	ch := make(chan *Response, 1)
+	c.pending[id] = ch
+	c.pmu.Unlock()
+
+	cleanup := func() {
+		c.pmu.Lock()
+		delete(c.pending, id)
+		c.pmu.Unlock()
+	}
+
+	err := c.write(frameKindRequest, frameHeader{
+		ID:      id,
+		Method:  method,
+		Target:  target,
+		Headers: map[string][]string(headers),
+	}, body)
+	if err != nil {
+		cleanup()
+		return nil, err
+	}
+
+	select {
+	case resp, ok := <-ch:
+		cleanup()
+		if !ok || resp == nil {
+			return nil, ErrTunnelClosed
+		}
+		return resp, nil
+	case <-ctx.Done():
+		cleanup()
+		return nil, ctx.Err()
+	case <-c.closed:
+		cleanup()
+		return nil, ErrTunnelClosed
+	}
+}
+
+// serveRequest executes an incoming request frame against handler and writes
+// the response frame back.
+func (c *tunnelConn) serveRequest(handler http.Handler, hdr frameHeader, body []byte) {
+	u, err := url.ParseRequestURI(hdr.Target)
+	if err != nil || !strings.HasPrefix(hdr.Target, "/") {
+		c.write(frameKindResponse, frameHeader{ID: hdr.ID, Status: http.StatusBadRequest}, []byte("invalid tunnel target"))
+		return
+	}
+	req, err := http.NewRequest(hdr.Method, hdr.Target, bytes.NewReader(body))
+	if err != nil {
+		c.write(frameKindResponse, frameHeader{ID: hdr.ID, Status: http.StatusBadRequest}, []byte(err.Error()))
+		return
+	}
+	req.URL = u
+	req.RequestURI = hdr.Target
+	req.RemoteAddr = "tunnel"
+	for k, vals := range hdr.Headers {
+		for _, v := range vals {
+			req.Header.Add(k, v)
+		}
+	}
+	req.ContentLength = int64(len(body))
+
+	rec := &responseRecorder{header: http.Header{}, status: http.StatusOK}
+	handler.ServeHTTP(rec, req)
+	c.write(frameKindResponse, frameHeader{
+		ID:      hdr.ID,
+		Status:  rec.status,
+		Headers: map[string][]string(rec.header),
+	}, rec.body.Bytes())
+}
+
+// readLoop pumps frames until the socket dies. Response frames complete
+// pending round trips; request frames are served through handler (nil when
+// this end never expects requests).
+func (c *tunnelConn) readLoop(handler http.Handler) {
+	defer c.close()
+	c.ws.SetReadDeadline(time.Now().Add(TunnelReadTimeout))
+	c.ws.SetPongHandler(func(string) error {
+		c.ws.SetReadDeadline(time.Now().Add(TunnelReadTimeout))
+		return nil
+	})
+	c.ws.SetPingHandler(func(data string) error {
+		c.ws.SetReadDeadline(time.Now().Add(TunnelReadTimeout))
+		c.writeMu.Lock()
+		defer c.writeMu.Unlock()
+		return c.ws.WriteControl(websocket.PongMessage, []byte(data), time.Now().Add(10*time.Second))
+	})
+
+	for {
+		mt, data, err := c.ws.ReadMessage()
+		if err != nil {
+			if !c.isClosed() && !websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
+				logger.PrintAndLog("Cluster", "Tunnel read ended: "+err.Error(), nil)
+			}
+			return
+		}
+		c.ws.SetReadDeadline(time.Now().Add(TunnelReadTimeout))
+		if mt != websocket.BinaryMessage {
+			continue
+		}
+		kind, hdr, body, err := decodeFrame(data)
+		if err != nil {
+			continue
+		}
+		switch kind {
+		case frameKindResponse:
+			c.pmu.Lock()
+			ch, ok := c.pending[hdr.ID]
+			c.pmu.Unlock()
+			if ok {
+				ch <- &Response{Status: hdr.Status, Header: http.Header(hdr.Headers), Body: body}
+			}
+		case frameKindRequest:
+			if handler != nil {
+				go c.serveRequest(handler, hdr, body)
+			}
+		}
+	}
+}
+
+// pingLoop keeps the socket alive through proxies until it closes.
+func (c *tunnelConn) pingLoop() {
+	ticker := time.NewTicker(TunnelPingInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-c.closed:
+			return
+		case <-ticker.C:
+			if err := c.ping(); err != nil {
+				c.close()
+				return
+			}
+		}
+	}
+}
+
+// responseRecorder captures a handler's output for the response frame.
+type responseRecorder struct {
+	header http.Header
+	status int
+	body   bytes.Buffer
+	wrote  bool
+}
+
+func (r *responseRecorder) Header() http.Header { return r.header }
+func (r *responseRecorder) Write(b []byte) (int, error) {
+	r.wrote = true
+	return r.body.Write(b)
+}
+func (r *responseRecorder) WriteHeader(status int) {
+	if !r.wrote {
+		r.status = status
+	}
+}
+
+/*
+	Host side
+*/
+
+// TunnelHub terminates tunnels from NAT-only nodes on a reachable node.
+type TunnelHub struct {
+	Verifier     *Verifier
+	OnConnect    func(nodeID string)
+	OnDisconnect func(nodeID string)
+
+	mu       sync.RWMutex
+	conns    map[string]*tunnelConn
+	upgrader websocket.Upgrader
+}
+
+// NewTunnelHub creates a hub whose incoming tunnels are authenticated by v.
+func NewTunnelHub(v *Verifier) *TunnelHub {
+	return &TunnelHub{
+		Verifier: v,
+		conns:    map[string]*tunnelConn{},
+		upgrader: websocket.Upgrader{
+			ReadBufferSize:  32 * 1024,
+			WriteBufferSize: 32 * 1024,
+			CheckOrigin:     func(r *http.Request) bool { return true },
+		},
+	}
+}
+
+// HandleTunnel is the HTTP handler for TunnelPath. The upgrade request must be
+// signed by a cluster member.
+func (h *TunnelHub) HandleTunnel(w http.ResponseWriter, r *http.Request) {
+	identity, _, err := h.Verifier.VerifyRequest(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusUnauthorized)
+		return
+	}
+	ws, err := h.upgrader.Upgrade(w, r, nil)
+	if err != nil {
+		return
+	}
+	conn := newTunnelConn(ws)
+
+	h.mu.Lock()
+	if old, exists := h.conns[identity.NodeID]; exists {
+		old.close()
+	}
+	h.conns[identity.NodeID] = conn
+	h.mu.Unlock()
+	logger.PrintAndLog("Cluster", "Tunnel established from node "+identity.NodeID, nil)
+	if h.OnConnect != nil {
+		h.OnConnect(identity.NodeID)
+	}
+
+	go conn.pingLoop()
+	conn.readLoop(nil)
+
+	h.mu.Lock()
+	if cur, exists := h.conns[identity.NodeID]; exists && cur == conn {
+		delete(h.conns, identity.NodeID)
+	}
+	h.mu.Unlock()
+	logger.PrintAndLog("Cluster", "Tunnel from node "+identity.NodeID+" closed", nil)
+	if h.OnDisconnect != nil {
+		h.OnDisconnect(identity.NodeID)
+	}
+}
+
+// Connected reports whether nodeID currently has a live tunnel here.
+func (h *TunnelHub) Connected(nodeID string) bool {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	c, ok := h.conns[nodeID]
+	return ok && !c.isClosed()
+}
+
+// ConnectedNodes lists the node IDs with a live tunnel.
+func (h *TunnelHub) ConnectedNodes() []string {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	ids := []string{}
+	for id, c := range h.conns {
+		if !c.isClosed() {
+			ids = append(ids, id)
+		}
+	}
+	return ids
+}
+
+// Do forwards an already signed request to the tunnelled node.
+func (h *TunnelHub) Do(ctx context.Context, nodeID string, method string, target string, headers http.Header, body []byte) (*Response, error) {
+	h.mu.RLock()
+	c, ok := h.conns[nodeID]
+	h.mu.RUnlock()
+	if !ok || c.isClosed() {
+		return nil, ErrTunnelNotConnected
+	}
+	return c.roundTrip(ctx, method, target, headers, body)
+}
+
+// Close drops every tunnel.
+func (h *TunnelHub) Close() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	for id, c := range h.conns {
+		c.close()
+		delete(h.conns, id)
+	}
+}
+
+/*
+	Client side
+*/
+
+// TunnelHost describes the reachable peer a NAT-only node should tunnel through.
+type TunnelHost struct {
+	NodeID string
+	URL    string
+}
+
+// TunnelClient keeps a NAT-only node attached to a reachable peer.
+type TunnelClient struct {
+	Signer  *Signer
+	Handler http.Handler //Local ACN handler that serves the multiplexed requests
+	//PickHost returns the peer to connect to. ok is false when no host is available yet.
+	PickHost      func() (TunnelHost, bool)
+	InsecureTLS   bool
+	OnStateChange func(connected bool, host TunnelHost)
+
+	mu        sync.Mutex
+	stop      chan struct{}
+	running   bool
+	connected bool
+	host      TunnelHost
+	current   *tunnelConn
+	dialer    func(host TunnelHost) (*websocket.Conn, error)
+}
+
+// Start begins the connect / serve / reconnect loop in the background.
+func (c *TunnelClient) Start() {
+	c.mu.Lock()
+	if c.running {
+		c.mu.Unlock()
+		return
+	}
+	c.running = true
+	c.stop = make(chan struct{})
+	stop := c.stop
+	c.mu.Unlock()
+	go c.loop(stop)
+}
+
+// Stop closes the current tunnel and halts reconnection.
+func (c *TunnelClient) Stop() {
+	c.mu.Lock()
+	if !c.running {
+		c.mu.Unlock()
+		return
+	}
+	c.running = false
+	close(c.stop)
+	cur := c.current
+	c.mu.Unlock()
+	if cur != nil {
+		cur.close()
+	}
+}
+
+// Reconnect drops the current tunnel so the loop picks a host again, used
+// after the preferred host changes.
+func (c *TunnelClient) Reconnect() {
+	c.mu.Lock()
+	cur := c.current
+	c.mu.Unlock()
+	if cur != nil {
+		cur.close()
+	}
+}
+
+// Status reports whether the tunnel is up and through which host.
+func (c *TunnelClient) Status() (bool, TunnelHost) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.connected, c.host
+}
+
+func (c *TunnelClient) setState(connected bool, host TunnelHost, conn *tunnelConn) {
+	c.mu.Lock()
+	c.connected = connected
+	c.host = host
+	c.current = conn
+	cb := c.OnStateChange
+	c.mu.Unlock()
+	if cb != nil {
+		cb(connected, host)
+	}
+}
+
+func (c *TunnelClient) dial(host TunnelHost) (*websocket.Conn, error) {
+	if c.dialer != nil {
+		return c.dialer(host)
+	}
+	wsURL, err := WebSocketURL(host.URL, TunnelPath)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequest(http.MethodGet, wsURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	c.Signer.Sign(req, TunnelPath, []byte{})
+	dialer := websocket.Dialer{
+		HandshakeTimeout: 20 * time.Second,
+		TLSClientConfig:  tlsConfig(c.InsecureTLS),
+	}
+	ws, resp, err := dialer.Dial(wsURL, req.Header)
+	if err != nil {
+		if resp != nil && resp.Body != nil {
+			msg, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
+			resp.Body.Close()
+			if len(msg) > 0 {
+				return nil, errors.New(err.Error() + ": " + strings.TrimSpace(string(msg)))
+			}
+		}
+		return nil, err
+	}
+	return ws, nil
+}
+
+func (c *TunnelClient) loop(stop chan struct{}) {
+	backoff := 2 * time.Second
+	for {
+		select {
+		case <-stop:
+			return
+		default:
+		}
+
+		host, ok := c.PickHost()
+		if !ok {
+			if !sleepOrStop(stop, 5*time.Second) {
+				return
+			}
+			continue
+		}
+
+		ws, err := c.dial(host)
+		if err != nil {
+			logger.PrintAndLog("Cluster", "Tunnel to "+host.NodeID+" failed: "+err.Error(), nil)
+			if !sleepOrStop(stop, backoff) {
+				return
+			}
+			if backoff < 60*time.Second {
+				backoff *= 2
+			}
+			continue
+		}
+		backoff = 2 * time.Second
+		conn := newTunnelConn(ws)
+		c.setState(true, host, conn)
+		logger.PrintAndLog("Cluster", "Tunnel connected through node "+host.NodeID, nil)
+		go conn.pingLoop()
+		conn.readLoop(c.Handler) //blocks until the socket dies
+		c.setState(false, host, nil)
+		logger.PrintAndLog("Cluster", "Tunnel through node "+host.NodeID+" disconnected", nil)
+		if !sleepOrStop(stop, backoff) {
+			return
+		}
+	}
+}
+
+func sleepOrStop(stop chan struct{}, d time.Duration) bool {
+	select {
+	case <-stop:
+		return false
+	case <-time.After(d):
+		return true
+	}
+}
+
+// WebSocketURL converts a node base URL plus path into a ws:// or wss:// URL.
+func WebSocketURL(baseURL string, path string) (string, error) {
+	u, err := url.Parse(strings.TrimRight(baseURL, "/"))
+	if err != nil {
+		return "", err
+	}
+	switch u.Scheme {
+	case "https":
+		u.Scheme = "wss"
+	case "http":
+		u.Scheme = "ws"
+	case "ws", "wss":
+	default:
+		return "", errors.New("unsupported node URL scheme: " + u.Scheme)
+	}
+	u.Path = strings.TrimRight(u.Path, "/") + path
+	u.RawQuery = ""
+	return u.String(), nil
+}

src/mod/cluster/capability/capability.go

@@ -0,0 +1,143 @@
+package capability
+
+/*
+	ArozOS Cluster - node capability manifest
+
+	Cluster nodes are heterogeneous (different OS, CPU architecture, RAM,
+	accelerators and installed tools). Each node publishes a manifest in its
+	membership record so the job scheduler can match a job's requirements
+	against what a node can actually do.
+
+	Detection is deliberately portable: it only uses runtime information,
+	the Go CPU feature flags and PATH lookups, never platform-only commands.
+*/
+
+import (
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"time"
+
+	"golang.org/x/sys/cpu"
+	"imuslab.com/arozos/mod/info/usageinfo"
+)
+
+// Manifest describes what a node offers.
+type Manifest struct {
+	OS         string          `json:"os"`
+	Arch       string          `json:"arch"`
+	CPUCores   int             `json:"cpuCores"`
+	TotalRAM   int64           `json:"totalRam"` //bytes, 0 when unknown
+	Hostname   string          `json:"hostname"`
+	Features   map[string]bool `json:"features"`
+	GoVersion  string          `json:"goVersion"`
+	DetectedAt int64           `json:"detectedAt"`
+}
+
+// Requirements is what a job asks for. Empty fields mean "no constraint".
+type Requirements struct {
+	Features []string `json:"features,omitempty"`
+	MinRAM   int64    `json:"minRam,omitempty"`
+	MinCores int      `json:"minCores,omitempty"`
+	OS       []string `json:"os,omitempty"`
+	Arch     []string `json:"arch,omitempty"`
+}
+
+// toolFeatures maps a feature name to the executable whose presence enables it.
+var toolFeatures = map[string]string{
+	"ffmpeg":  "ffmpeg",
+	"ffprobe": "ffprobe",
+	"docker":  "docker",
+	"nvidia":  "nvidia-smi",
+	"git":     "git",
+	"python3": "python3",
+	"node":    "node",
+}
+
+// Detect builds the manifest of the local machine.
+func Detect() Manifest {
+	m := Manifest{
+		OS:         runtime.GOOS,
+		Arch:       runtime.GOARCH,
+		CPUCores:   runtime.NumCPU(),
+		Features:   map[string]bool{},
+		GoVersion:  runtime.Version(),
+		DetectedAt: time.Now().Unix(),
+	}
+	if hn, err := os.Hostname(); err == nil {
+		m.Hostname = hn
+	}
+	if _, total := usageinfo.GetNumericRAMUsage(); total > 0 {
+		m.TotalRAM = total
+	}
+
+	for feature, binary := range toolFeatures {
+		m.Features[feature] = lookPath(binary)
+	}
+	if m.Features["nvidia"] {
+		m.Features["cuda"] = true
+		m.Features["gpu"] = true
+	}
+
+	//CPU instruction sets, false on architectures that do not have them
+	m.Features["avx"] = cpu.X86.HasAVX
+	m.Features["avx2"] = cpu.X86.HasAVX2
+	m.Features["avx512"] = cpu.X86.HasAVX512F
+	m.Features["neon"] = cpu.ARM64.HasASIMD
+	m.Features["64bit"] = strings.HasSuffix(runtime.GOARCH, "64")
+	return m
+}
+
+func lookPath(binary string) bool {
+	_, err := exec.LookPath(binary)
+	return err == nil
+}
+
+// Has reports whether a feature is present.
+func (m Manifest) Has(feature string) bool {
+	return m.Features != nil && m.Features[strings.ToLower(feature)]
+}
+
+// Satisfies checks the manifest against job requirements and explains the
+// first unmet one.
+func (m Manifest) Satisfies(r Requirements) (bool, string) {
+	for _, f := range r.Features {
+		if !m.Has(f) {
+			return false, "missing feature " + f
+		}
+	}
+	if r.MinRAM > 0 && m.TotalRAM > 0 && m.TotalRAM < r.MinRAM {
+		return false, "insufficient memory"
+	}
+	if r.MinCores > 0 && m.CPUCores < r.MinCores {
+		return false, "insufficient cpu cores"
+	}
+	if len(r.OS) > 0 && !containsFold(r.OS, m.OS) {
+		return false, "os " + m.OS + " not allowed"
+	}
+	if len(r.Arch) > 0 && !containsFold(r.Arch, m.Arch) {
+		return false, "architecture " + m.Arch + " not allowed"
+	}
+	return true, ""
+}
+
+// EnabledFeatures returns the sorted-insensitive list of features that are true.
+func (m Manifest) EnabledFeatures() []string {
+	out := []string{}
+	for f, on := range m.Features {
+		if on {
+			out = append(out, f)
+		}
+	}
+	return out
+}
+
+func containsFold(list []string, v string) bool {
+	for _, item := range list {
+		if strings.EqualFold(item, v) {
+			return true
+		}
+	}
+	return false
+}

src/mod/cluster/capability/capability_test.go

@@ -0,0 +1,84 @@
+package capability
+
+import (
+	"runtime"
+	"testing"
+)
+
+func TestDetectBasics(t *testing.T) {
+	m := Detect()
+	if m.OS != runtime.GOOS || m.Arch != runtime.GOARCH {
+		t.Errorf("os/arch mismatch: %s/%s", m.OS, m.Arch)
+	}
+	if m.CPUCores < 1 {
+		t.Errorf("expected at least one core, got %d", m.CPUCores)
+	}
+	if m.Features == nil {
+		t.Fatal("features map missing")
+	}
+	for _, f := range []string{"ffmpeg", "docker", "avx2", "neon", "64bit"} {
+		if _, ok := m.Features[f]; !ok {
+			t.Errorf("feature %s should always be reported", f)
+		}
+	}
+	if m.DetectedAt == 0 {
+		t.Errorf("DetectedAt not set")
+	}
+}
+
+func TestSatisfies(t *testing.T) {
+	m := Manifest{
+		OS:       "linux",
+		Arch:     "arm64",
+		CPUCores: 4,
+		TotalRAM: 8 << 30,
+		Features: map[string]bool{"ffmpeg": true, "docker": true, "cuda": false},
+	}
+	tests := []struct {
+		name string
+		req  Requirements
+		want bool
+	}{
+		{"empty", Requirements{}, true},
+		{"has features", Requirements{Features: []string{"ffmpeg", "docker"}}, true},
+		{"missing feature", Requirements{Features: []string{"cuda"}}, false},
+		{"unknown feature", Requirements{Features: []string{"quantum"}}, false},
+		{"ram ok", Requirements{MinRAM: 4 << 30}, true},
+		{"ram too small", Requirements{MinRAM: 16 << 30}, false},
+		{"cores ok", Requirements{MinCores: 4}, true},
+		{"cores too few", Requirements{MinCores: 8}, false},
+		{"os allowed", Requirements{OS: []string{"Linux", "darwin"}}, true},
+		{"os denied", Requirements{OS: []string{"windows"}}, false},
+		{"arch allowed", Requirements{Arch: []string{"amd64", "ARM64"}}, true},
+		{"arch denied", Requirements{Arch: []string{"amd64"}}, false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ok, reason := m.Satisfies(tc.req)
+			if ok != tc.want {
+				t.Errorf("Satisfies(%+v) = %v (%s), want %v", tc.req, ok, reason, tc.want)
+			}
+			if !ok && reason == "" {
+				t.Errorf("unmet requirement must carry a reason")
+			}
+		})
+	}
+}
+
+func TestSatisfiesUnknownRAM(t *testing.T) {
+	m := Manifest{TotalRAM: 0, CPUCores: 2, Features: map[string]bool{}}
+	if ok, _ := m.Satisfies(Requirements{MinRAM: 1 << 40}); !ok {
+		t.Errorf("unknown RAM must not fail a RAM requirement")
+	}
+}
+
+func TestEnabledFeatures(t *testing.T) {
+	m := Manifest{Features: map[string]bool{"a": true, "b": false, "c": true}}
+	got := m.EnabledFeatures()
+	if len(got) != 2 {
+		t.Errorf("expected 2 enabled features, got %v", got)
+	}
+	if !m.Has("A") || m.Has("b") || m.Has("zzz") {
+		t.Errorf("Has() lookup wrong")
+	}
+}

src/mod/cluster/capability/diskusage_other.go

@@ -0,0 +1,10 @@
+//go:build !linux && !darwin && !freebsd && !windows
+
+package capability
+
+import "errors"
+
+// DiskUsage is not available on this platform; callers treat 0 as unknown.
+func DiskUsage(path string) (free int64, total int64, err error) {
+	return 0, 0, errors.New("disk usage not supported on this platform")
+}

src/mod/cluster/capability/diskusage_test.go

@@ -0,0 +1,27 @@
+package capability
+
+import (
+	"runtime"
+	"testing"
+)
+
+func TestDiskUsage(t *testing.T) {
+	dir := t.TempDir()
+	free, total, err := DiskUsage(dir)
+	switch runtime.GOOS {
+	case "linux", "darwin", "freebsd", "windows":
+		if err != nil {
+			t.Fatalf("DiskUsage(%s): %v", dir, err)
+		}
+		if total <= 0 || free < 0 || free > total {
+			t.Errorf("implausible disk figures free=%d total=%d", free, total)
+		}
+	default:
+		if err == nil {
+			t.Errorf("unsupported platform should return an error")
+		}
+	}
+	if _, _, err := DiskUsage(dir + "/does/not/exist"); err == nil && runtime.GOOS != "windows" {
+		t.Errorf("missing path should fail")
+	}
+}

src/mod/cluster/capability/diskusage_unix.go

@@ -0,0 +1,15 @@
+//go:build linux || darwin || freebsd
+
+package capability
+
+import "golang.org/x/sys/unix"
+
+// DiskUsage returns the free and total bytes of the file system holding path.
+func DiskUsage(path string) (free int64, total int64, err error) {
+	var st unix.Statfs_t
+	if err := unix.Statfs(path, &st); err != nil {
+		return 0, 0, err
+	}
+	bsize := int64(st.Bsize)
+	return int64(st.Bavail) * bsize, int64(st.Blocks) * bsize, nil
+}

src/mod/cluster/capability/diskusage_windows.go

@@ -0,0 +1,18 @@
+//go:build windows
+
+package capability
+
+import "golang.org/x/sys/windows"
+
+// DiskUsage returns the free and total bytes of the volume holding path.
+func DiskUsage(path string) (free int64, total int64, err error) {
+	p, err := windows.UTF16PtrFromString(path)
+	if err != nil {
+		return 0, 0, err
+	}
+	var freeAvail, totalBytes, totalFree uint64
+	if err := windows.GetDiskFreeSpaceEx(p, &freeAvail, &totalBytes, &totalFree); err != nil {
+		return 0, 0, err
+	}
+	return int64(freeAvail), int64(totalBytes), nil
+}

src/mod/cluster/membership/handlers_acn.go

@@ -0,0 +1,174 @@
+package membership
+
+/*
+	ArozOS Cluster - node-facing (ACN) endpoints
+
+	POST /cluster/acn/join           token-authenticated, joins a new node
+	POST /cluster/acn/heartbeat      signed, exchanges liveness and membership
+	GET  /cluster/acn/members        signed, full membership dump
+	POST /cluster/acn/members/sync   signed, push of membership changes
+	POST /cluster/acn/leave          signed, sender leaves
+	POST /cluster/acn/evict          signed, sender removed this node
+*/
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+	"time"
+
+	"imuslab.com/arozos/mod/cluster/acn"
+	"imuslab.com/arozos/mod/info/logger"
+)
+
+func (m *Manager) registerACNHandlers() {
+	m.server.HandleRaw(acn.BasePath+"/join", m.handleJoin)
+	m.server.HandleFunc(acn.BasePath+"/heartbeat", m.handleHeartbeat)
+	m.server.HandleFunc(acn.BasePath+"/members", m.handleMembers)
+	m.server.HandleFunc(acn.BasePath+"/members/sync", m.handleSync)
+	m.server.HandleFunc(acn.BasePath+"/leave", m.handleLeave)
+	m.server.HandleFunc(acn.BasePath+"/evict", m.handleEvict)
+}
+
+func sanitizeName(name string) string {
+	name = strings.TrimSpace(name)
+	if len(name) > 64 {
+		name = name[:64]
+	}
+	return name
+}
+
+func (m *Manager) handleJoin(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		acn.WriteError(w, http.StatusMethodNotAllowed, "POST required")
+		return
+	}
+	body, err := acn.ReadBody(r)
+	if err != nil {
+		acn.WriteError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	var req JoinRequest
+	if err := json.Unmarshal(body, &req); err != nil {
+		acn.WriteError(w, http.StatusBadRequest, "invalid join request")
+		return
+	}
+
+	m.mu.Lock()
+	if m.cluster == nil {
+		m.mu.Unlock()
+		acn.WriteError(w, http.StatusNotFound, "this node is not part of a cluster")
+		return
+	}
+	if req.ClusterID != m.cluster.ID {
+		m.mu.Unlock()
+		acn.WriteError(w, http.StatusForbidden, "join token is for another cluster")
+		return
+	}
+	token, ok := m.tokens[req.TokenID]
+	if !ok || !token.Valid(req.Secret, time.Now()) {
+		m.mu.Unlock()
+		acn.WriteError(w, http.StatusForbidden, "join token is invalid or expired")
+		return
+	}
+	node := req.Node
+	node.Name = sanitizeName(node.Name)
+	if node.ID == "" || node.ID == m.opt.NodeID || node.Name == "" {
+		m.mu.Unlock()
+		acn.WriteError(w, http.StatusBadRequest, "invalid node record")
+		return
+	}
+	if _, err := acn.DecodePublicKey(node.PublicKey); err != nil {
+		m.mu.Unlock()
+		acn.WriteError(w, http.StatusBadRequest, "invalid node public key")
+		return
+	}
+	node.AdvertiseURL = strings.TrimRight(strings.TrimSpace(node.AdvertiseURL), "/")
+	if node.AdvertiseURL != "" && !strings.HasPrefix(node.AdvertiseURL, "http://") && !strings.HasPrefix(node.AdvertiseURL, "https://") {
+		m.mu.Unlock()
+		acn.WriteError(w, http.StatusBadRequest, "invalid node URL")
+		return
+	}
+	now := time.Now().Unix()
+	node.Removed = false
+	node.AdminState = AdminStateNormal
+	node.Joined = now
+	node.Updated = nextVersion(0)
+	node.LastSeen = now
+	node.TunnelVia = ""
+	if node.AdvertiseURL == "" {
+		node.TunnelVia = m.opt.NodeID
+	}
+	m.nodes[node.ID] = &node
+	m.store.saveNode(&node)
+	token.Uses++
+	m.store.saveJoinToken(token)
+	resp := JoinResponse{Cluster: *m.cluster, Nodes: m.allRecordsLocked(), Via: m.opt.NodeID}
+	peers := m.peerIDsLocked()
+	m.mu.Unlock()
+
+	logger.PrintAndLog("Cluster", "Node "+node.Name+" ("+node.ID+") joined the cluster", nil)
+	acn.WriteJSON(w, resp)
+
+	others := []string{}
+	for _, id := range peers {
+		if id != node.ID {
+			others = append(others, id)
+		}
+	}
+	go m.broadcast(others, acn.BasePath+"/members/sync", SyncRequest{Nodes: m.snapshotRecords()}, 15*time.Second)
+}
+
+func (m *Manager) handleHeartbeat(w http.ResponseWriter, r *http.Request, sender *acn.SignedIdentity, body []byte) {
+	var req HeartbeatRequest
+	if err := json.Unmarshal(body, &req); err != nil || req.Node.ID != sender.NodeID {
+		acn.WriteError(w, http.StatusBadRequest, "invalid heartbeat")
+		return
+	}
+	req.Node.LastSeen = time.Now().Unix()
+	m.mergeNodes([]NodeRecord{req.Node})
+	m.markSeen(sender.NodeID)
+	acn.WriteJSON(w, HeartbeatResponse{Nodes: m.snapshotRecords(), Time: time.Now().Unix()})
+}
+
+func (m *Manager) handleMembers(w http.ResponseWriter, r *http.Request, sender *acn.SignedIdentity, body []byte) {
+	m.markSeen(sender.NodeID)
+	acn.WriteJSON(w, SyncRequest{Nodes: m.snapshotRecords()})
+}
+
+func (m *Manager) handleSync(w http.ResponseWriter, r *http.Request, sender *acn.SignedIdentity, body []byte) {
+	var req SyncRequest
+	if err := json.Unmarshal(body, &req); err != nil {
+		acn.WriteError(w, http.StatusBadRequest, "invalid sync payload")
+		return
+	}
+	m.markSeen(sender.NodeID)
+	m.mergeNodes(req.Nodes)
+	acn.WriteJSON(w, map[string]bool{"ok": true})
+}
+
+func (m *Manager) handleLeave(w http.ResponseWriter, r *http.Request, sender *acn.SignedIdentity, body []byte) {
+	m.mu.Lock()
+	rec, ok := m.nodes[sender.NodeID]
+	if ok {
+		rec.Removed = true
+		rec.Updated = nextVersion(rec.Updated)
+		m.store.saveNode(rec)
+	}
+	m.mu.Unlock()
+	if ok {
+		logger.PrintAndLog("Cluster", "Node "+rec.Name+" ("+rec.ID+") left the cluster", nil)
+	}
+	acn.WriteJSON(w, map[string]bool{"ok": true})
+}
+
+func (m *Manager) handleEvict(w http.ResponseWriter, r *http.Request, sender *acn.SignedIdentity, body []byte) {
+	var req NodeIDRequest
+	if err := json.Unmarshal(body, &req); err != nil || req.NodeID != m.opt.NodeID {
+		acn.WriteError(w, http.StatusBadRequest, "evict notice is not for this node")
+		return
+	}
+	acn.WriteJSON(w, map[string]bool{"ok": true})
+	logger.PrintAndLog("Cluster", "Removed from the cluster by node "+sender.NodeID, nil)
+	go m.wipeLocalState()
+}

src/mod/cluster/membership/handlers_admin.go

@@ -0,0 +1,241 @@
+package membership
+
+/*
+	ArozOS Cluster - admin (System Settings) endpoints
+
+	These handlers are mounted by the core under /system/cluster/* through the
+	permission router with AdminOnly set, so they never check auth themselves.
+*/
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"strings"
+	"time"
+
+	"imuslab.com/arozos/mod/cluster/acn"
+	"imuslab.com/arozos/mod/utils"
+)
+
+func sendJSON(w http.ResponseWriter, v interface{}) {
+	js, _ := json.Marshal(v)
+	utils.SendJSONResponse(w, string(js))
+}
+
+// HandleStatus returns the whole cluster picture for the settings page.
+func (m *Manager) HandleStatus(w http.ResponseWriter, r *http.Request) {
+	sendJSON(w, m.Status())
+}
+
+// HandleCreate creates a new cluster with this node as first member.
+func (m *Manager) HandleCreate(w http.ResponseWriter, r *http.Request) {
+	name, err := utils.PostPara(r, "name")
+	if err != nil {
+		utils.SendErrorResponse(w, "cluster name required")
+		return
+	}
+	info, err := m.CreateCluster(name)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+	sendJSON(w, info)
+}
+
+// HandleJoin joins the cluster described by a pasted join token.
+func (m *Manager) HandleJoin(w http.ResponseWriter, r *http.Request) {
+	token, err := utils.PostPara(r, "token")
+	if err != nil {
+		utils.SendErrorResponse(w, "join token required")
+		return
+	}
+	info, err := m.JoinCluster(token)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+	sendJSON(w, info)
+}
+
+// HandleLeave leaves the current cluster.
+func (m *Manager) HandleLeave(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		utils.SendErrorResponse(w, "POST required")
+		return
+	}
+	if err := m.LeaveCluster(); err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+	utils.SendOK(w)
+}
+
+// HandleConfig reads (GET) or updates (POST) the local node configuration.
+func (m *Manager) HandleConfig(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		sendJSON(w, m.Config())
+		return
+	}
+	cfg := m.Config()
+	if name, err := utils.PostPara(r, "name"); err == nil {
+		cfg.Name = name
+	}
+	if url, err := utils.PostPara(r, "url"); err == nil {
+		cfg.AdvertiseURL = url
+	} else if r.PostFormValue("url") == "" && r.PostForm.Has("url") {
+		cfg.AdvertiseURL = ""
+	}
+	if via, err := utils.PostPara(r, "tunnelvia"); err == nil {
+		cfg.TunnelVia = via
+	} else if r.PostForm.Has("tunnelvia") {
+		cfg.TunnelVia = ""
+	}
+	if insecure, err := utils.PostBool(r, "insecure"); err == nil {
+		cfg.InsecureTLS = insecure
+	}
+	if err := m.UpdateConfig(cfg); err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+	sendJSON(w, m.Config())
+}
+
+// HandleTestURL checks that a URL reaches THIS node's ACN endpoint, which is
+// how an admin validates the advertised URL before saving it.
+func (m *Manager) HandleTestURL(w http.ResponseWriter, r *http.Request) {
+	url, err := utils.PostPara(r, "url")
+	if err != nil {
+		utils.SendErrorResponse(w, "url required")
+		return
+	}
+	url = strings.TrimRight(strings.TrimSpace(url), "/")
+	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
+		utils.SendErrorResponse(w, "URL must start with http:// or https://")
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	start := time.Now()
+	resp, err := m.transport.DoURL(ctx, url, http.MethodGet, acn.BasePath+"/hello", nil, false)
+	if err != nil {
+		utils.SendErrorResponse(w, "unreachable: "+err.Error())
+		return
+	}
+	var hello acn.HelloResponse
+	if resp.Status != http.StatusOK || json.Unmarshal(resp.Body, &hello) != nil || !hello.ACN {
+		utils.SendErrorResponse(w, "the URL answered but it is not an ArozOS cluster endpoint")
+		return
+	}
+	sendJSON(w, map[string]interface{}{
+		"ok":        true,
+		"latencyMs": time.Since(start).Milliseconds(),
+		"version":   hello.Version,
+		"inCluster": hello.InCluster,
+	})
+}
+
+// HandleTokenNew issues a join token. Optional ttl in hours (default 24).
+func (m *Manager) HandleTokenNew(w http.ResponseWriter, r *http.Request) {
+	ttl := 24 * time.Hour
+	if hours, err := utils.PostInt(r, "ttl"); err == nil && hours > 0 && hours <= 24*30 {
+		ttl = time.Duration(hours) * time.Hour
+	}
+	encoded, token, err := m.NewJoinToken(ttl)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+	sendJSON(w, map[string]interface{}{"token": encoded, "id": token.ID, "expires": token.Expires})
+}
+
+// HandleTokenList lists active join tokens.
+func (m *Manager) HandleTokenList(w http.ResponseWriter, r *http.Request) {
+	sendJSON(w, m.ListJoinTokens())
+}
+
+// HandleTokenRevoke deletes a join token.
+func (m *Manager) HandleTokenRevoke(w http.ResponseWriter, r *http.Request) {
+	id, err := utils.PostPara(r, "id")
+	if err != nil {
+		utils.SendErrorResponse(w, "token id required")
+		return
+	}
+	if err := m.RevokeJoinToken(id); err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+	utils.SendOK(w)
+}
+
+// HandleNodeRemove evicts a member.
+func (m *Manager) HandleNodeRemove(w http.ResponseWriter, r *http.Request) {
+	id, err := utils.PostPara(r, "id")
+	if err != nil {
+		utils.SendErrorResponse(w, "node id required")
+		return
+	}
+	if err := m.RemoveNode(id); err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+	utils.SendOK(w)
+}
+
+// HandleNodeState sets a member to normal / maintenance / draining.
+func (m *Manager) HandleNodeState(w http.ResponseWriter, r *http.Request) {
+	id, err := utils.PostPara(r, "id")
+	if err != nil {
+		utils.SendErrorResponse(w, "node id required")
+		return
+	}
+	state, _ := utils.PostPara(r, "state")
+	if err := m.SetNodeAdminState(id, state); err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+	utils.SendOK(w)
+}
+
+// HandleNodeProbe pings a member on demand.
+func (m *Manager) HandleNodeProbe(w http.ResponseWriter, r *http.Request) {
+	id, err := utils.GetPara(r, "id")
+	if err != nil {
+		utils.SendErrorResponse(w, "node id required")
+		return
+	}
+	sendJSON(w, m.ProbeNode(id))
+}
+
+// HandleCapabilities returns (and with refresh=true re-detects) the local manifest.
+func (m *Manager) HandleCapabilities(w http.ResponseWriter, r *http.Request) {
+	if refresh, _ := utils.GetBool(r, "refresh"); refresh {
+		sendJSON(w, m.RefreshCapabilities())
+		return
+	}
+	sendJSON(w, m.Status().Local.Capabilities)
+}
+
+// HandleNodes returns just the member list, handy for other settings pages.
+func (m *Manager) HandleNodes(w http.ResponseWriter, r *http.Request) {
+	sendJSON(w, m.NodeViews())
+}
+
+// RegisterAdminRoutes mounts every admin handler on the given register
+// function (typically a permission router's HandleFunc).
+func (m *Manager) RegisterAdminRoutes(register func(pattern string, handler func(http.ResponseWriter, *http.Request))) {
+	register("/system/cluster/status", m.HandleStatus)
+	register("/system/cluster/create", m.HandleCreate)
+	register("/system/cluster/join", m.HandleJoin)
+	register("/system/cluster/leave", m.HandleLeave)
+	register("/system/cluster/config", m.HandleConfig)
+	register("/system/cluster/testurl", m.HandleTestURL)
+	register("/system/cluster/token/new", m.HandleTokenNew)
+	register("/system/cluster/token/list", m.HandleTokenList)
+	register("/system/cluster/token/revoke", m.HandleTokenRevoke)
+	register("/system/cluster/node/remove", m.HandleNodeRemove)
+	register("/system/cluster/node/state", m.HandleNodeState)
+	register("/system/cluster/node/probe", m.HandleNodeProbe)
+	register("/system/cluster/nodes", m.HandleNodes)
+	register("/system/cluster/capabilities", m.HandleCapabilities)
+}

src/mod/cluster/membership/jointoken.go

@@ -0,0 +1,104 @@
+package membership
+
+/*
+	ArozOS Cluster - join tokens
+
+	A join token is generated on a reachable member and pasted into the
+	System Settings of the node that wants to join. It carries everything the
+	joiner needs: the cluster ID and name, the URL of the issuing node and a
+	one-time-style secret. Only the SHA-256 of the secret is stored server
+	side.
+*/
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"strings"
+	"time"
+)
+
+const joinTokenPrefix = "aroz-join:"
+
+// joinTokenPayload is the wire form embedded in the pasted string.
+type joinTokenPayload struct {
+	ClusterID   string `json:"c"`
+	ClusterName string `json:"n"`
+	URL         string `json:"u"`
+	TokenID     string `json:"t"`
+	Secret      string `json:"s"`
+}
+
+func randomHex(n int) string {
+	buf := make([]byte, n)
+	rand.Read(buf)
+	return hex.EncodeToString(buf)
+}
+
+func hashSecret(secret string) string {
+	sum := sha256.Sum256([]byte(secret))
+	return hex.EncodeToString(sum[:])
+}
+
+// NewJoinToken creates a token valid for ttl and the pasteable string that
+// encodes it for the issuing node at issuerURL.
+func NewJoinToken(cluster ClusterInfo, issuerURL string, ttl time.Duration) (*JoinToken, string, error) {
+	if strings.TrimSpace(issuerURL) == "" {
+		return nil, "", errors.New("this node has no advertised URL; generate the join token on a node that other nodes can reach")
+	}
+	secret := randomHex(32)
+	now := time.Now()
+	token := &JoinToken{
+		ID:         randomHex(8),
+		SecretHash: hashSecret(secret),
+		Created:    now.Unix(),
+		Expires:    now.Add(ttl).Unix(),
+	}
+	payload := joinTokenPayload{
+		ClusterID:   cluster.ID,
+		ClusterName: cluster.Name,
+		URL:         strings.TrimRight(strings.TrimSpace(issuerURL), "/"),
+		TokenID:     token.ID,
+		Secret:      secret,
+	}
+	js, err := json.Marshal(payload)
+	if err != nil {
+		return nil, "", err
+	}
+	return token, joinTokenPrefix + base64.RawURLEncoding.EncodeToString(js), nil
+}
+
+// DecodeJoinToken parses a pasted token string.
+func DecodeJoinToken(encoded string) (*joinTokenPayload, error) {
+	encoded = strings.TrimSpace(encoded)
+	if !strings.HasPrefix(encoded, joinTokenPrefix) {
+		return nil, errors.New("not an ArozOS join token")
+	}
+	raw, err := base64.RawURLEncoding.DecodeString(strings.TrimPrefix(encoded, joinTokenPrefix))
+	if err != nil {
+		return nil, errors.New("join token is corrupted")
+	}
+	var p joinTokenPayload
+	if err := json.Unmarshal(raw, &p); err != nil {
+		return nil, errors.New("join token is corrupted")
+	}
+	if p.ClusterID == "" || p.URL == "" || p.TokenID == "" || p.Secret == "" {
+		return nil, errors.New("join token is incomplete")
+	}
+	if !strings.HasPrefix(p.URL, "http://") && !strings.HasPrefix(p.URL, "https://") {
+		return nil, errors.New("join token has an invalid node URL")
+	}
+	return &p, nil
+}
+
+// Valid checks a presented secret against the stored token at time now.
+func (t *JoinToken) Valid(secret string, now time.Time) bool {
+	if t == nil || now.Unix() > t.Expires {
+		return false
+	}
+	return subtle.ConstantTimeCompare([]byte(hashSecret(secret)), []byte(t.SecretHash)) == 1
+}

src/mod/cluster/membership/manager.go

@@ -0,0 +1,1048 @@
+package membership
+
+/*
+	ArozOS Cluster - membership manager
+
+	The Manager is the cluster agent of one node. It owns the node key, the
+	cluster database, the ACN server/transport and keeps the replicated
+	membership view converged with the other nodes through heartbeats.
+
+	Life cycle of a node:
+
+		standalone --CreateCluster/JoinCluster--> member --LeaveCluster/evicted--> standalone
+
+	Every node stays a fully working standalone ArozOS whatever its state.
+*/
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	uuid "github.com/satori/go.uuid"
+	"imuslab.com/arozos/mod/cluster/acn"
+	"imuslab.com/arozos/mod/cluster/capability"
+	"imuslab.com/arozos/mod/info/logger"
+)
+
+// Option configures a Manager.
+type Option struct {
+	NodeID       string
+	DBFile       string
+	KeyFile      string
+	Version      string
+	DefaultName  string
+	Capabilities func() capability.Manifest
+	Health       func() Health
+}
+
+// Manager is the cluster agent of this node.
+type Manager struct {
+	opt   Option
+	store *store
+	key   *acn.NodeKey
+
+	mu      sync.RWMutex
+	cluster *ClusterInfo
+	nodes   map[string]*NodeRecord
+	tokens  map[string]*JoinToken
+	config  LocalConfig
+	caps    capability.Manifest
+
+	signer    *acn.Signer
+	verifier  *acn.Verifier
+	hub       *acn.TunnelHub
+	server    *acn.Server
+	transport *acn.Transport
+	tunnel    *acn.TunnelClient
+
+	loopMu    sync.Mutex
+	loopStop  chan struct{}
+	loopWG    sync.WaitGroup
+	reachable map[string]bool //last known reachability per peer, for log de-duplication
+	started   time.Time
+}
+
+var (
+	ErrNotInCluster     = errors.New("this node is not part of a cluster")
+	ErrAlreadyInCluster = errors.New("this node is already part of a cluster, leave it first")
+	ErrNodeNotFound     = errors.New("node not found")
+)
+
+// NewManager opens the cluster database and node key and, when the node was
+// part of a cluster before the restart, resumes membership.
+func NewManager(opt Option) (*Manager, error) {
+	if opt.NodeID == "" {
+		return nil, errors.New("node id is required")
+	}
+	if opt.Capabilities == nil {
+		opt.Capabilities = capability.Detect
+	}
+	if opt.Health == nil {
+		opt.Health = func() Health { return Health{} }
+	}
+	st, err := newStore(opt.DBFile)
+	if err != nil {
+		return nil, err
+	}
+	key, err := acn.LoadOrCreateNodeKey(opt.KeyFile)
+	if err != nil {
+		st.close()
+		return nil, err
+	}
+
+	m := &Manager{
+		opt:       opt,
+		store:     st,
+		key:       key,
+		nodes:     map[string]*NodeRecord{},
+		tokens:    map[string]*JoinToken{},
+		reachable: map[string]bool{},
+		started:   time.Now(),
+	}
+	m.caps = opt.Capabilities()
+	m.config = st.loadConfig()
+	if m.config.Name == "" {
+		m.config.Name = opt.DefaultName
+	}
+	m.cluster = st.loadCluster()
+	m.nodes = st.loadNodes()
+	m.tokens = st.loadJoinTokens()
+
+	clusterID := ""
+	if m.cluster != nil {
+		clusterID = m.cluster.ID
+	}
+	m.signer = &acn.Signer{NodeID: opt.NodeID, ClusterID: clusterID, Key: key}
+	m.verifier = acn.NewVerifier(m.clusterID, m)
+	m.hub = acn.NewTunnelHub(m.verifier)
+	m.hub.OnConnect = m.onTunnelConnect
+	m.server = acn.NewServer(m.verifier, m.hub, opt.Version)
+	m.registerACNHandlers()
+	m.transport = acn.NewTransport(m.signer, m, m.hub, m.config.InsecureTLS)
+	m.tunnel = &acn.TunnelClient{
+		Signer:        m.signer,
+		Handler:       m.server,
+		PickHost:      m.pickTunnelHost,
+		InsecureTLS:   m.config.InsecureTLS,
+		OnStateChange: m.onTunnelState,
+	}
+
+	m.mu.Lock()
+	m.refreshLocalRecordLocked()
+	m.mu.Unlock()
+
+	if m.cluster != nil {
+		logger.PrintAndLog("Cluster", "Resuming membership of cluster "+m.cluster.Name+" ("+m.cluster.ID+")", nil)
+		m.startLoops()
+	}
+	return m, nil
+}
+
+// Close stops heartbeats and tunnels and closes the database.
+func (m *Manager) Close() {
+	m.stopLoops()
+	m.hub.Close()
+	m.store.close()
+}
+
+// ACNHandler is the HTTP handler to mount at acn.BasePath.
+func (m *Manager) ACNHandler() http.Handler {
+	return m.server
+}
+
+// Transport exposes the node-to-node transport to higher layers.
+func (m *Manager) Transport() *acn.Transport {
+	return m.transport
+}
+
+// Server exposes the ACN server so higher layers can register endpoints.
+func (m *Manager) Server() *acn.Server {
+	return m.server
+}
+
+// NodeID returns the ID of this node.
+func (m *Manager) NodeID() string {
+	return m.opt.NodeID
+}
+
+// InCluster reports whether this node is currently a cluster member.
+func (m *Manager) InCluster() bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.cluster != nil
+}
+
+// Cluster returns a copy of the cluster info, nil when standalone.
+func (m *Manager) Cluster() *ClusterInfo {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	if m.cluster == nil {
+		return nil
+	}
+	c := *m.cluster
+	return &c
+}
+
+func (m *Manager) clusterID() string {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	if m.cluster == nil {
+		return ""
+	}
+	return m.cluster.ID
+}
+
+// ResolvePeer implements acn.PeerResolver.
+func (m *Manager) ResolvePeer(nodeID string) (*acn.Peer, bool) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	rec, ok := m.nodes[nodeID]
+	if !ok || rec.Removed {
+		return nil, false
+	}
+	pub, err := acn.DecodePublicKey(rec.PublicKey)
+	if err != nil {
+		return nil, false
+	}
+	return &acn.Peer{
+		ID:           rec.ID,
+		Name:         rec.Name,
+		PublicKey:    pub,
+		AdvertiseURL: rec.AdvertiseURL,
+		TunnelVia:    rec.TunnelVia,
+	}, true
+}
+
+/*
+	Local record
+*/
+
+// refreshLocalRecordLocked makes sure the record describing this node exists
+// and reflects the current config, key and capabilities.
+func (m *Manager) refreshLocalRecordLocked() {
+	now := time.Now().Unix()
+	rec, ok := m.nodes[m.opt.NodeID]
+	if !ok {
+		rec = &NodeRecord{ID: m.opt.NodeID, Joined: now}
+		m.nodes[m.opt.NodeID] = rec
+	}
+	changed := !ok || rec.Removed
+	url := strings.TrimRight(strings.TrimSpace(m.config.AdvertiseURL), "/")
+	if rec.Name != m.config.Name || rec.PublicKey != m.key.PublicKeyString() || rec.AdvertiseURL != url || rec.Version != m.opt.Version {
+		changed = true
+	}
+	if rec.Capabilities.DetectedAt != m.caps.DetectedAt {
+		changed = true
+	}
+	if url != "" && rec.TunnelVia != "" {
+		//A reachable node never needs a tunnel host
+		rec.TunnelVia = ""
+		changed = true
+	}
+	if changed {
+		rec.Name = m.config.Name
+		rec.PublicKey = m.key.PublicKeyString()
+		rec.AdvertiseURL = url
+		rec.Version = m.opt.Version
+		rec.Capabilities = m.caps
+		rec.Removed = false
+		rec.Updated = nextVersion(rec.Updated)
+		m.store.saveNode(rec)
+	}
+}
+
+// localSnapshotLocked returns the local record with fresh transient fields.
+func (m *Manager) localSnapshotLocked() NodeRecord {
+	rec := m.nodes[m.opt.NodeID].Clone()
+	h := m.opt.Health()
+	h.Uptime = int64(time.Since(m.started).Seconds())
+	h.Timestamp = time.Now().Unix()
+	rec.Health = h
+	rec.LastSeen = h.Timestamp
+	return *rec
+}
+
+// allRecordsLocked lists every record including tombstones, self first.
+func (m *Manager) allRecordsLocked() []NodeRecord {
+	out := []NodeRecord{m.localSnapshotLocked()}
+	ids := make([]string, 0, len(m.nodes))
+	for id := range m.nodes {
+		if id != m.opt.NodeID {
+			ids = append(ids, id)
+		}
+	}
+	sort.Strings(ids)
+	for _, id := range ids {
+		out = append(out, *m.nodes[id].Clone())
+	}
+	return out
+}
+
+func (m *Manager) peerIDsLocked() []string {
+	ids := []string{}
+	for id, rec := range m.nodes {
+		if id != m.opt.NodeID && !rec.Removed {
+			ids = append(ids, id)
+		}
+	}
+	sort.Strings(ids)
+	return ids
+}
+
+/*
+	Merge
+*/
+
+// mergeRecordLocked folds one gossiped record into the local view. It reports
+// whether persistent fields changed and whether this node has been evicted.
+func (m *Manager) mergeRecordLocked(in NodeRecord) (changed bool, evicted bool) {
+	if in.ID == "" {
+		return false, false
+	}
+	if in.ID == m.opt.NodeID {
+		local := m.nodes[m.opt.NodeID]
+		if in.Updated > local.Updated {
+			if in.Removed {
+				return false, true
+			}
+			//Only admin state may be set on us by other members
+			if in.AdminState != local.AdminState {
+				local.AdminState = in.AdminState
+				local.Updated = in.Updated
+				return true, false
+			}
+		}
+		return false, false
+	}
+
+	existing, ok := m.nodes[in.ID]
+	if !ok {
+		rec := in
+		m.nodes[in.ID] = &rec
+		return true, false
+	}
+	if in.Updated > existing.Updated {
+		existing.Name = in.Name
+		existing.PublicKey = in.PublicKey
+		existing.AdvertiseURL = in.AdvertiseURL
+		existing.TunnelVia = in.TunnelVia
+		existing.Version = in.Version
+		existing.Capabilities = in.Capabilities
+		existing.AdminState = in.AdminState
+		existing.Removed = in.Removed
+		existing.Joined = in.Joined
+		existing.Updated = in.Updated
+		changed = true
+	}
+	if in.Health.Timestamp > existing.Health.Timestamp {
+		existing.Health = in.Health
+	}
+	if in.LastSeen > existing.LastSeen {
+		existing.LastSeen = in.LastSeen
+	}
+	return changed, false
+}
+
+// mergeNodes applies a batch of records, persists changes and handles eviction.
+func (m *Manager) mergeNodes(records []NodeRecord) {
+	m.mu.Lock()
+	if m.cluster == nil {
+		m.mu.Unlock()
+		return
+	}
+	evicted := false
+	for _, in := range records {
+		changed, ev := m.mergeRecordLocked(in)
+		if ev {
+			evicted = true
+		}
+		if changed {
+			m.store.saveNode(m.nodes[in.ID])
+		}
+	}
+	m.gcTombstonesLocked()
+	m.mu.Unlock()
+
+	if evicted {
+		logger.PrintAndLog("Cluster", "This node has been removed from the cluster by another member", nil)
+		m.wipeLocalState()
+	}
+}
+
+func (m *Manager) gcTombstonesLocked() {
+	cutoff := time.Now().Add(-TombstoneTTL).UnixMilli()
+	for id, rec := range m.nodes {
+		if rec.Removed && rec.Updated < cutoff && id != m.opt.NodeID {
+			delete(m.nodes, id)
+			m.store.deleteNode(id)
+		}
+	}
+}
+
+/*
+	Cluster life cycle
+*/
+
+// CreateCluster turns this standalone node into the first member of a new cluster.
+func (m *Manager) CreateCluster(name string) (*ClusterInfo, error) {
+	name = strings.TrimSpace(name)
+	if name == "" {
+		return nil, errors.New("cluster name cannot be empty")
+	}
+	m.mu.Lock()
+	if m.cluster != nil {
+		m.mu.Unlock()
+		return nil, ErrAlreadyInCluster
+	}
+	info := &ClusterInfo{ID: uuid.NewV4().String(), Name: name, Created: time.Now().Unix()}
+	if err := m.store.saveCluster(info); err != nil {
+		m.mu.Unlock()
+		return nil, err
+	}
+	m.cluster = info
+	m.signer.ClusterID = info.ID
+	m.nodes = map[string]*NodeRecord{}
+	m.refreshLocalRecordLocked()
+	local := m.nodes[m.opt.NodeID]
+	local.Joined = info.Created
+	local.Updated = nextVersion(local.Updated)
+	m.store.saveNode(local)
+	m.mu.Unlock()
+
+	logger.PrintAndLog("Cluster", "Created cluster "+name+" ("+info.ID+")", nil)
+	m.startLoops()
+	c := *info
+	return &c, nil
+}
+
+// JoinCluster contacts the issuing node named in the token and joins its cluster.
+func (m *Manager) JoinCluster(tokenString string) (*ClusterInfo, error) {
+	payload, err := DecodeJoinToken(tokenString)
+	if err != nil {
+		return nil, err
+	}
+	m.mu.Lock()
+	if m.cluster != nil {
+		m.mu.Unlock()
+		return nil, ErrAlreadyInCluster
+	}
+	m.refreshLocalRecordLocked()
+	now := time.Now().Unix()
+	local := m.nodes[m.opt.NodeID].Clone()
+	local.Joined = now
+	local.Updated = nextVersion(local.Updated)
+	local.AdminState = AdminStateNormal
+	req := JoinRequest{ClusterID: payload.ClusterID, TokenID: payload.TokenID, Secret: payload.Secret, Node: *local}
+	m.mu.Unlock()
+
+	js, _ := json.Marshal(req)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	resp, err := m.transport.DoURL(ctx, payload.URL, http.MethodPost, acn.BasePath+"/join", js, false)
+	if err != nil {
+		return nil, errors.New("unable to reach " + payload.URL + ": " + err.Error())
+	}
+	if err := resp.Error(); err != nil {
+		return nil, err
+	}
+	var jr JoinResponse
+	if err := json.Unmarshal(resp.Body, &jr); err != nil {
+		return nil, errors.New("invalid join response from " + payload.URL)
+	}
+	if jr.Cluster.ID == "" || jr.Cluster.ID != payload.ClusterID {
+		return nil, errors.New("join response does not match the token's cluster")
+	}
+
+	m.mu.Lock()
+	if m.cluster != nil {
+		m.mu.Unlock()
+		return nil, ErrAlreadyInCluster
+	}
+	info := jr.Cluster
+	if err := m.store.saveCluster(&info); err != nil {
+		m.mu.Unlock()
+		return nil, err
+	}
+	m.cluster = &info
+	m.signer.ClusterID = info.ID
+	m.nodes = map[string]*NodeRecord{}
+	self := *local
+	if self.AdvertiseURL == "" {
+		self.TunnelVia = jr.Via
+	}
+	m.nodes[self.ID] = &self
+	m.store.saveNode(&self)
+	for _, rec := range jr.Nodes {
+		if changed, _ := m.mergeRecordLocked(rec); changed {
+			m.store.saveNode(m.nodes[rec.ID])
+		}
+	}
+	m.mu.Unlock()
+
+	logger.PrintAndLog("Cluster", "Joined cluster "+info.Name+" ("+info.ID+") through "+payload.URL, nil)
+	m.startLoops()
+	return &info, nil
+}
+
+// LeaveCluster tells the other members goodbye and returns to standalone mode.
+func (m *Manager) LeaveCluster() error {
+	m.mu.RLock()
+	if m.cluster == nil {
+		m.mu.RUnlock()
+		return ErrNotInCluster
+	}
+	peers := m.peerIDsLocked()
+	m.mu.RUnlock()
+
+	m.stopLoops()
+	m.broadcast(peers, acn.BasePath+"/leave", NodeIDRequest{NodeID: m.opt.NodeID}, 10*time.Second)
+	m.wipeLocalState()
+	logger.PrintAndLog("Cluster", "Left the cluster", nil)
+	return nil
+}
+
+// wipeLocalState drops all cluster records and returns to standalone mode.
+func (m *Manager) wipeLocalState() {
+	m.stopLoops()
+	m.hub.Close()
+	m.mu.Lock()
+	m.store.wipeCluster()
+	m.cluster = nil
+	m.signer.ClusterID = ""
+	m.nodes = map[string]*NodeRecord{}
+	m.tokens = map[string]*JoinToken{}
+	m.reachable = map[string]bool{}
+	m.refreshLocalRecordLocked()
+	m.mu.Unlock()
+}
+
+// RemoveNode evicts another member.
+func (m *Manager) RemoveNode(nodeID string) error {
+	if nodeID == m.opt.NodeID {
+		return errors.New("use leave to remove this node")
+	}
+	m.mu.Lock()
+	if m.cluster == nil {
+		m.mu.Unlock()
+		return ErrNotInCluster
+	}
+	rec, ok := m.nodes[nodeID]
+	if !ok || rec.Removed {
+		m.mu.Unlock()
+		return ErrNodeNotFound
+	}
+	rec.Removed = true
+	rec.Updated = nextVersion(rec.Updated)
+	m.store.saveNode(rec)
+	peers := m.peerIDsLocked()
+	m.mu.Unlock()
+
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		//Tell the evicted node itself first; ResolvePeer no longer knows it, so resolve manually
+		m.mu.RLock()
+		target := m.nodes[nodeID]
+		m.mu.RUnlock()
+		if target != nil {
+			m.notifyEvicted(ctx, target)
+		}
+		m.broadcast(peers, acn.BasePath+"/members/sync", SyncRequest{Nodes: m.snapshotRecords()}, 15*time.Second)
+	}()
+	logger.PrintAndLog("Cluster", "Removed node "+nodeID+" from the cluster", nil)
+	return nil
+}
+
+// notifyEvicted delivers the evict notice to a node that is no longer resolvable.
+func (m *Manager) notifyEvicted(ctx context.Context, target *NodeRecord) {
+	body, _ := json.Marshal(NodeIDRequest{NodeID: target.ID})
+	if target.AdvertiseURL != "" {
+		m.transport.DoURL(ctx, target.AdvertiseURL, http.MethodPost, acn.BasePath+"/evict", body, true)
+		return
+	}
+	if m.hub.Connected(target.ID) {
+		req, _ := http.NewRequest(http.MethodPost, acn.BasePath+"/evict", nil)
+		req.Header.Set("Content-Type", "application/json")
+		m.signer.Sign(req, acn.BasePath+"/evict", body)
+		m.hub.Do(ctx, target.ID, http.MethodPost, acn.BasePath+"/evict", req.Header, body)
+	}
+}
+
+// SetNodeAdminState puts a node into maintenance / draining / normal mode.
+func (m *Manager) SetNodeAdminState(nodeID string, state string) error {
+	state = strings.ToLower(strings.TrimSpace(state))
+	if state == "normal" {
+		state = AdminStateNormal
+	}
+	if state != AdminStateNormal && state != AdminStateMaintenance && state != AdminStateDraining {
+		return errors.New("unsupported node state")
+	}
+	m.mu.Lock()
+	if m.cluster == nil {
+		m.mu.Unlock()
+		return ErrNotInCluster
+	}
+	rec, ok := m.nodes[nodeID]
+	if !ok || rec.Removed {
+		m.mu.Unlock()
+		return ErrNodeNotFound
+	}
+	rec.AdminState = state
+	rec.Updated = nextVersion(rec.Updated)
+	m.store.saveNode(rec)
+	peers := m.peerIDsLocked()
+	m.mu.Unlock()
+	go m.broadcast(peers, acn.BasePath+"/members/sync", SyncRequest{Nodes: m.snapshotRecords()}, 15*time.Second)
+	return nil
+}
+
+/*
+	Configuration
+*/
+
+// Config returns the local node configuration.
+func (m *Manager) Config() LocalConfig {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.config
+}
+
+// UpdateConfig changes the local node configuration and republishes the record.
+func (m *Manager) UpdateConfig(cfg LocalConfig) error {
+	cfg.Name = strings.TrimSpace(cfg.Name)
+	cfg.AdvertiseURL = strings.TrimRight(strings.TrimSpace(cfg.AdvertiseURL), "/")
+	cfg.TunnelVia = strings.TrimSpace(cfg.TunnelVia)
+	if cfg.Name == "" {
+		return errors.New("node name cannot be empty")
+	}
+	if cfg.AdvertiseURL != "" && !strings.HasPrefix(cfg.AdvertiseURL, "http://") && !strings.HasPrefix(cfg.AdvertiseURL, "https://") {
+		return errors.New("advertise URL must start with http:// or https://")
+	}
+	if cfg.TunnelVia == m.opt.NodeID {
+		cfg.TunnelVia = ""
+	}
+
+	m.mu.Lock()
+	old := m.config
+	m.config = cfg
+	if err := m.store.saveConfig(cfg); err != nil {
+		m.config = old
+		m.mu.Unlock()
+		return err
+	}
+	m.transport.SetInsecureTLS(cfg.InsecureTLS)
+	m.tunnel.InsecureTLS = cfg.InsecureTLS
+	m.refreshLocalRecordLocked()
+	inCluster := m.cluster != nil
+	peers := m.peerIDsLocked()
+	m.mu.Unlock()
+
+	if inCluster {
+		switch {
+		case old.AdvertiseURL == "" && cfg.AdvertiseURL != "":
+			m.tunnel.Stop()
+		case old.AdvertiseURL != "" && cfg.AdvertiseURL == "":
+			m.tunnel.Start()
+		case cfg.AdvertiseURL == "" && old.TunnelVia != cfg.TunnelVia:
+			m.tunnel.Reconnect()
+		}
+		go m.broadcast(peers, acn.BasePath+"/members/sync", SyncRequest{Nodes: m.snapshotRecords()}, 15*time.Second)
+	}
+	return nil
+}
+
+// RefreshCapabilities re-detects the local manifest and republishes it.
+func (m *Manager) RefreshCapabilities() capability.Manifest {
+	caps := m.opt.Capabilities()
+	m.mu.Lock()
+	m.caps = caps
+	m.refreshLocalRecordLocked()
+	m.mu.Unlock()
+	return caps
+}
+
+/*
+	Join tokens
+*/
+
+// NewJoinToken issues a token other nodes can use to join through this node.
+func (m *Manager) NewJoinToken(ttl time.Duration) (string, *JoinToken, error) {
+	if ttl <= 0 {
+		ttl = 24 * time.Hour
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.cluster == nil {
+		return "", nil, ErrNotInCluster
+	}
+	token, encoded, err := NewJoinToken(*m.cluster, m.config.AdvertiseURL, ttl)
+	if err != nil {
+		return "", nil, err
+	}
+	if err := m.store.saveJoinToken(token); err != nil {
+		return "", nil, err
+	}
+	m.tokens[token.ID] = token
+	return encoded, token, nil
+}
+
+// ListJoinTokens returns the unexpired tokens, purging expired ones.
+func (m *Manager) ListJoinTokens() []JoinToken {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	now := time.Now().Unix()
+	out := []JoinToken{}
+	for id, t := range m.tokens {
+		if t.Expires < now {
+			delete(m.tokens, id)
+			m.store.deleteJoinToken(id)
+			continue
+		}
+		out = append(out, *t)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Created > out[j].Created })
+	return out
+}
+
+// RevokeJoinToken deletes a token before it expires.
+func (m *Manager) RevokeJoinToken(id string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if _, ok := m.tokens[id]; !ok {
+		return errors.New("token not found")
+	}
+	delete(m.tokens, id)
+	return m.store.deleteJoinToken(id)
+}
+
+/*
+	Views
+*/
+
+// NodeViews lists the live members with their computed state.
+func (m *Manager) NodeViews() []NodeView {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	now := time.Now()
+	out := []NodeView{}
+	for _, rec := range m.allRecordsLocked() {
+		if rec.Removed {
+			continue
+		}
+		local := rec.ID == m.opt.NodeID
+		out = append(out, NodeView{
+			NodeRecord: rec,
+			State:      rec.ComputeState(now, local),
+			Local:      local,
+			Tunnel:     m.hub.Connected(rec.ID),
+		})
+	}
+	return out
+}
+
+// snapshotRecords returns every record for gossip.
+func (m *Manager) snapshotRecords() []NodeRecord {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.allRecordsLocked()
+}
+
+// LocalView summarises this node for the settings UI.
+type LocalView struct {
+	NodeID          string              `json:"nodeId"`
+	PublicKey       string              `json:"publicKey"`
+	Config          LocalConfig         `json:"config"`
+	Version         string              `json:"version"`
+	Capabilities    capability.Manifest `json:"capabilities"`
+	Health          Health              `json:"health"`
+	Reachable       bool                `json:"reachable"`
+	TunnelConnected bool                `json:"tunnelConnected"`
+	TunnelHost      string              `json:"tunnelHost"`
+}
+
+// Status is the full picture handed to the admin UI.
+type Status struct {
+	InCluster bool         `json:"inCluster"`
+	Cluster   *ClusterInfo `json:"cluster"`
+	Local     LocalView    `json:"local"`
+	Nodes     []NodeView   `json:"nodes"`
+	Tokens    []JoinToken  `json:"tokens"`
+	Tunnels   []string     `json:"tunnels"`
+	Time      int64        `json:"time"`
+}
+
+// Status builds the admin view.
+func (m *Manager) Status() Status {
+	connected, host := m.tunnel.Status()
+	m.mu.RLock()
+	local := m.localSnapshotLocked()
+	cfg := m.config
+	var cluster *ClusterInfo
+	if m.cluster != nil {
+		c := *m.cluster
+		cluster = &c
+	}
+	m.mu.RUnlock()
+
+	return Status{
+		InCluster: cluster != nil,
+		Cluster:   cluster,
+		Local: LocalView{
+			NodeID:          m.opt.NodeID,
+			PublicKey:       m.key.PublicKeyString(),
+			Config:          cfg,
+			Version:         m.opt.Version,
+			Capabilities:    local.Capabilities,
+			Health:          local.Health,
+			Reachable:       cfg.AdvertiseURL != "",
+			TunnelConnected: connected,
+			TunnelHost:      host.NodeID,
+		},
+		Nodes:   m.NodeViews(),
+		Tokens:  m.ListJoinTokens(),
+		Tunnels: m.hub.ConnectedNodes(),
+		Time:    time.Now().Unix(),
+	}
+}
+
+// ProbeResult is the outcome of an on-demand reachability check.
+type ProbeResult struct {
+	NodeID    string `json:"nodeId"`
+	OK        bool   `json:"ok"`
+	LatencyMs int64  `json:"latencyMs"`
+	Route     string `json:"route"`
+	Error     string `json:"error,omitempty"`
+}
+
+// ProbeNode pings a member through whatever route the transport picks.
+func (m *Manager) ProbeNode(nodeID string) ProbeResult {
+	res := ProbeResult{NodeID: nodeID}
+	peer, ok := m.ResolvePeer(nodeID)
+	if !ok {
+		res.Error = ErrNodeNotFound.Error()
+		return res
+	}
+	switch {
+	case m.hub.Connected(nodeID):
+		res.Route = "tunnel"
+	case peer.AdvertiseURL != "":
+		res.Route = "direct"
+	case peer.TunnelVia != "":
+		res.Route = "relay via " + peer.TunnelVia
+	default:
+		res.Route = "none"
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	start := time.Now()
+	err := m.transport.DoJSON(ctx, nodeID, http.MethodPost, acn.BasePath+"/ping", nil, nil)
+	res.LatencyMs = time.Since(start).Milliseconds()
+	if err != nil {
+		res.Error = err.Error()
+		return res
+	}
+	res.OK = true
+	m.markSeen(nodeID)
+	return res
+}
+
+/*
+	Gossip loops
+*/
+
+func (m *Manager) startLoops() {
+	m.loopMu.Lock()
+	if m.loopStop != nil {
+		m.loopMu.Unlock()
+		return
+	}
+	m.loopStop = make(chan struct{})
+	stop := m.loopStop
+	m.loopWG.Add(1)
+	m.loopMu.Unlock()
+
+	go func() {
+		defer m.loopWG.Done()
+		m.heartbeatAll()
+		ticker := time.NewTicker(HeartbeatInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-stop:
+				return
+			case <-ticker.C:
+				m.heartbeatAll()
+			}
+		}
+	}()
+
+	if m.Config().AdvertiseURL == "" {
+		m.tunnel.Start()
+	}
+}
+
+func (m *Manager) stopLoops() {
+	m.loopMu.Lock()
+	if m.loopStop == nil {
+		m.loopMu.Unlock()
+		return
+	}
+	close(m.loopStop)
+	m.loopStop = nil
+	m.loopMu.Unlock()
+	m.loopWG.Wait()
+	m.tunnel.Stop()
+}
+
+// heartbeatAll reports to every peer in parallel and merges their views.
+func (m *Manager) heartbeatAll() {
+	m.mu.RLock()
+	if m.cluster == nil {
+		m.mu.RUnlock()
+		return
+	}
+	local := m.localSnapshotLocked()
+	peers := m.peerIDsLocked()
+	m.mu.RUnlock()
+
+	var wg sync.WaitGroup
+	for _, id := range peers {
+		wg.Add(1)
+		go func(peerID string) {
+			defer wg.Done()
+			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+			defer cancel()
+			var resp HeartbeatResponse
+			err := m.transport.DoJSON(ctx, peerID, http.MethodPost, acn.BasePath+"/heartbeat", HeartbeatRequest{Node: local}, &resp)
+			m.noteReachability(peerID, err)
+			if err != nil {
+				return
+			}
+			m.markSeen(peerID)
+			m.mergeNodes(resp.Nodes)
+		}(id)
+	}
+	wg.Wait()
+}
+
+func (m *Manager) markSeen(nodeID string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if rec, ok := m.nodes[nodeID]; ok {
+		rec.LastSeen = time.Now().Unix()
+	}
+}
+
+// noteReachability logs only when a peer flips between reachable and not.
+func (m *Manager) noteReachability(peerID string, err error) {
+	m.mu.Lock()
+	prev, known := m.reachable[peerID]
+	now := err == nil
+	m.reachable[peerID] = now
+	name := peerID
+	if rec, ok := m.nodes[peerID]; ok && rec.Name != "" {
+		name = rec.Name
+	}
+	m.mu.Unlock()
+	if !known || prev != now {
+		if now {
+			logger.PrintAndLog("Cluster", "Node "+name+" is reachable", nil)
+		} else {
+			logger.PrintAndLog("Cluster", "Node "+name+" is unreachable: "+err.Error(), nil)
+		}
+	}
+}
+
+// broadcast posts payload to the given peers in parallel and waits for all.
+func (m *Manager) broadcast(peers []string, path string, payload interface{}, timeout time.Duration) {
+	var wg sync.WaitGroup
+	for _, id := range peers {
+		wg.Add(1)
+		go func(peerID string) {
+			defer wg.Done()
+			ctx, cancel := context.WithTimeout(context.Background(), timeout)
+			defer cancel()
+			m.transport.DoJSON(ctx, peerID, http.MethodPost, path, payload, nil)
+		}(id)
+	}
+	wg.Wait()
+}
+
+/*
+	Tunnel callbacks
+*/
+
+// pickTunnelHost chooses the reachable peer a NAT-only node should attach to.
+func (m *Manager) pickTunnelHost() (acn.TunnelHost, bool) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	if m.cluster == nil {
+		return acn.TunnelHost{}, false
+	}
+	if pref := m.config.TunnelVia; pref != "" {
+		if rec, ok := m.nodes[pref]; ok && !rec.Removed && rec.AdvertiseURL != "" {
+			return acn.TunnelHost{NodeID: rec.ID, URL: rec.AdvertiseURL}, true
+		}
+	}
+	now := time.Now()
+	var best *NodeRecord
+	for _, id := range m.peerIDsLocked() {
+		rec := m.nodes[id]
+		if rec.AdvertiseURL == "" {
+			continue
+		}
+		if best == nil {
+			best = rec
+			continue
+		}
+		bs, rs := best.ComputeState(now, false), rec.ComputeState(now, false)
+		if bs != StateOnline && rs == StateOnline {
+			best = rec
+		} else if bs == rs && rec.LastSeen > best.LastSeen {
+			best = rec
+		}
+	}
+	if best == nil {
+		return acn.TunnelHost{}, false
+	}
+	return acn.TunnelHost{NodeID: best.ID, URL: best.AdvertiseURL}, true
+}
+
+// onTunnelState records which host now terminates our tunnel and tells it.
+func (m *Manager) onTunnelState(connected bool, host acn.TunnelHost) {
+	if !connected {
+		return
+	}
+	m.mu.Lock()
+	local, ok := m.nodes[m.opt.NodeID]
+	if ok && local.TunnelVia != host.NodeID {
+		local.TunnelVia = host.NodeID
+		local.Updated = nextVersion(local.Updated)
+		m.store.saveNode(local)
+	}
+	peers := m.peerIDsLocked()
+	m.mu.Unlock()
+	go m.broadcast(peers, acn.BasePath+"/members/sync", SyncRequest{Nodes: m.snapshotRecords()}, 15*time.Second)
+}
+
+// onTunnelConnect runs on the host side when a NAT-only node attaches.
+func (m *Manager) onTunnelConnect(nodeID string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	rec, ok := m.nodes[nodeID]
+	if !ok {
+		return
+	}
+	rec.LastSeen = time.Now().Unix()
+	if rec.TunnelVia != m.opt.NodeID {
+		rec.TunnelVia = m.opt.NodeID
+		rec.Updated = nextVersion(rec.Updated)
+		m.store.saveNode(rec)
+	}
+}

src/mod/cluster/membership/membership_test.go

@@ -0,0 +1,437 @@
+package membership
+
+import (
+	"net/http/httptest"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"imuslab.com/arozos/mod/cluster/capability"
+)
+
+/*
+	Helpers
+*/
+
+type testCluster struct {
+	t *testing.T
+}
+
+// newTestManager spins up a manager with its own database, key and HTTP
+// server. When reachable is true the server URL is advertised so other nodes
+// can contact it directly; otherwise the node is NAT-only and must tunnel.
+func newTestManager(t *testing.T, id string, reachable bool) (*Manager, *httptest.Server) {
+	t.Helper()
+	dir := t.TempDir()
+	m, err := NewManager(Option{
+		NodeID:      id,
+		DBFile:      filepath.Join(dir, "cluster.db"),
+		KeyFile:     filepath.Join(dir, "node.key"),
+		Version:     "test",
+		DefaultName: "Node " + id,
+		Capabilities: func() capability.Manifest {
+			return capability.Manifest{OS: "testos", Arch: "testarch", CPUCores: 2, Features: map[string]bool{"ffmpeg": true}, DetectedAt: 1}
+		},
+		Health: func() Health { return Health{CPUUsage: 10, RAMUsed: 1, RAMTotal: 4} },
+	})
+	if err != nil {
+		t.Fatalf("NewManager(%s): %v", id, err)
+	}
+	srv := httptest.NewServer(m.ACNHandler())
+	if reachable {
+		cfg := m.Config()
+		cfg.AdvertiseURL = srv.URL
+		if err := m.UpdateConfig(cfg); err != nil {
+			t.Fatalf("UpdateConfig(%s): %v", id, err)
+		}
+	}
+	t.Cleanup(func() {
+		m.Close()
+		srv.Close()
+	})
+	return m, srv
+}
+
+func waitFor(t *testing.T, what string, timeout time.Duration, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for !cond() {
+		if time.Now().After(deadline) {
+			t.Fatalf("timed out waiting for %s", what)
+		}
+		time.Sleep(25 * time.Millisecond)
+	}
+}
+
+func memberIDs(m *Manager) map[string]NodeView {
+	out := map[string]NodeView{}
+	for _, v := range m.NodeViews() {
+		out[v.ID] = v
+	}
+	return out
+}
+
+/*
+	Join tokens
+*/
+
+func TestJoinTokenRoundTrip(t *testing.T) {
+	cluster := ClusterInfo{ID: "c1", Name: "Home"}
+	token, encoded, err := NewJoinToken(cluster, "https://a.example.com/", time.Hour)
+	if err != nil {
+		t.Fatalf("NewJoinToken: %v", err)
+	}
+	p, err := DecodeJoinToken(encoded)
+	if err != nil {
+		t.Fatalf("DecodeJoinToken: %v", err)
+	}
+	if p.ClusterID != "c1" || p.ClusterName != "Home" || p.URL != "https://a.example.com" || p.TokenID != token.ID {
+		t.Errorf("payload mismatch: %+v", p)
+	}
+	if !token.Valid(p.Secret, time.Now()) {
+		t.Errorf("token should validate its own secret")
+	}
+	if token.Valid("wrong", time.Now()) {
+		t.Errorf("wrong secret accepted")
+	}
+	if token.Valid(p.Secret, time.Now().Add(2*time.Hour)) {
+		t.Errorf("expired token accepted")
+	}
+}
+
+func TestJoinTokenInvalid(t *testing.T) {
+	if _, _, err := NewJoinToken(ClusterInfo{ID: "c"}, "", time.Hour); err == nil {
+		t.Errorf("token without issuer URL must fail")
+	}
+	for _, bad := range []string{"", "hello", "aroz-join:!!!", "aroz-join:e30"} {
+		if _, err := DecodeJoinToken(bad); err == nil {
+			t.Errorf("DecodeJoinToken(%q) expected error", bad)
+		}
+	}
+}
+
+/*
+	State machine
+*/
+
+func TestComputeState(t *testing.T) {
+	now := time.Now()
+	tests := []struct {
+		name  string
+		rec   NodeRecord
+		local bool
+		want  NodeState
+	}{
+		{"local always online", NodeRecord{}, true, StateOnline},
+		{"never seen", NodeRecord{}, false, StateUnknown},
+		{"fresh", NodeRecord{LastSeen: now.Add(-5 * time.Second).Unix()}, false, StateOnline},
+		{"fresh but loaded", NodeRecord{LastSeen: now.Unix(), Health: Health{CPUUsage: 99, Timestamp: now.Unix()}}, false, StateDegraded},
+		{"stale", NodeRecord{LastSeen: now.Add(-OnlineWindow - time.Second).Unix()}, false, StateUnknown},
+		{"gone", NodeRecord{LastSeen: now.Add(-OfflineWindow - time.Second).Unix()}, false, StateOffline},
+		{"maintenance wins", NodeRecord{AdminState: AdminStateMaintenance, LastSeen: now.Unix()}, false, StateMaintenance},
+		{"draining wins locally", NodeRecord{AdminState: AdminStateDraining}, true, StateDraining},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := tc.rec.ComputeState(now, tc.local); got != tc.want {
+				t.Errorf("got %s want %s", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestHealthDegraded(t *testing.T) {
+	tests := []struct {
+		name string
+		h    Health
+		want bool
+	}{
+		{"empty", Health{}, false},
+		{"normal", Health{CPUUsage: 40, RAMUsed: 2, RAMTotal: 8, DiskFree: 50, DiskTotal: 100, Timestamp: 1}, false},
+		{"cpu", Health{CPUUsage: 98, Timestamp: 1}, true},
+		{"ram", Health{RAMUsed: 98, RAMTotal: 100, Timestamp: 1}, true},
+		{"disk", Health{DiskFree: 1, DiskTotal: 100, Timestamp: 1}, true},
+	}
+	for _, tc := range tests {
+		if got := tc.h.Degraded(); got != tc.want {
+			t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
+		}
+	}
+}
+
+/*
+	Merge
+*/
+
+func TestMergeLastWriterWins(t *testing.T) {
+	m, _ := newTestManager(t, "self", true)
+	if _, err := m.CreateCluster("merge"); err != nil {
+		t.Fatalf("CreateCluster: %v", err)
+	}
+
+	m.mu.Lock()
+	changed, _ := m.mergeRecordLocked(NodeRecord{ID: "p", Name: "old", Updated: 10, LastSeen: 5, Health: Health{CPUUsage: 1, Timestamp: 5}})
+	if !changed {
+		t.Errorf("new record should be a change")
+	}
+	changed, _ = m.mergeRecordLocked(NodeRecord{ID: "p", Name: "stale", Updated: 5, LastSeen: 9, Health: Health{CPUUsage: 2, Timestamp: 9}})
+	if changed || m.nodes["p"].Name != "old" {
+		t.Errorf("older Updated must not overwrite static fields")
+	}
+	if m.nodes["p"].LastSeen != 9 || m.nodes["p"].Health.CPUUsage != 2 {
+		t.Errorf("transient fields must merge by recency")
+	}
+	changed, _ = m.mergeRecordLocked(NodeRecord{ID: "p", Name: "new", Updated: 20})
+	if !changed || m.nodes["p"].Name != "new" {
+		t.Errorf("newer Updated must win")
+	}
+
+	//Our own record: only admin state may be pushed onto us, and a removal evicts us
+	self := m.nodes["self"]
+	_, evicted := m.mergeRecordLocked(NodeRecord{ID: "self", Name: "hijack", Updated: self.Updated + 1})
+	if evicted || m.nodes["self"].Name == "hijack" {
+		t.Errorf("other nodes must not rename us")
+	}
+	changed, _ = m.mergeRecordLocked(NodeRecord{ID: "self", AdminState: AdminStateMaintenance, Updated: self.Updated + 2})
+	if !changed || m.nodes["self"].AdminState != AdminStateMaintenance {
+		t.Errorf("admin state should be accepted")
+	}
+	_, evicted = m.mergeRecordLocked(NodeRecord{ID: "self", Removed: true, Updated: self.Updated + 3})
+	if !evicted {
+		t.Errorf("newer tombstone for ourselves must evict")
+	}
+	m.mu.Unlock()
+}
+
+/*
+	Full life cycle over HTTP
+*/
+
+func TestCreateJoinHeartbeatLeave(t *testing.T) {
+	a, _ := newTestManager(t, "node-a", true)
+	b, _ := newTestManager(t, "node-b", true)
+
+	if a.InCluster() {
+		t.Fatalf("fresh node must be standalone")
+	}
+	if _, _, err := a.NewJoinToken(time.Hour); err != ErrNotInCluster {
+		t.Errorf("token before cluster: %v", err)
+	}
+	info, err := a.CreateCluster("Home Lab")
+	if err != nil {
+		t.Fatalf("CreateCluster: %v", err)
+	}
+	if _, err := a.CreateCluster("again"); err != ErrAlreadyInCluster {
+		t.Errorf("second create: %v", err)
+	}
+
+	token, _, err := a.NewJoinToken(time.Hour)
+	if err != nil {
+		t.Fatalf("NewJoinToken: %v", err)
+	}
+	joined, err := b.JoinCluster(token)
+	if err != nil {
+		t.Fatalf("JoinCluster: %v", err)
+	}
+	if joined.ID != info.ID {
+		t.Errorf("joined wrong cluster")
+	}
+	if len(a.ListJoinTokens()) != 1 || a.ListJoinTokens()[0].Uses != 1 {
+		t.Errorf("token usage not counted: %+v", a.ListJoinTokens())
+	}
+
+	waitFor(t, "a to see b online", 5*time.Second, func() bool {
+		v, ok := memberIDs(a)["node-b"]
+		return ok && v.State == StateOnline
+	})
+	waitFor(t, "b to see a online", 5*time.Second, func() bool {
+		v, ok := memberIDs(b)["node-a"]
+		return ok && v.State == StateOnline && v.Health.CPUUsage == 10
+	})
+	if len(a.NodeViews()) != 2 || len(b.NodeViews()) != 2 {
+		t.Fatalf("expected 2 members each, got %d / %d", len(a.NodeViews()), len(b.NodeViews()))
+	}
+
+	//Probe route
+	res := a.ProbeNode("node-b")
+	if !res.OK || res.Route != "direct" {
+		t.Errorf("probe b: %+v", res)
+	}
+
+	//Admin state propagates
+	if err := a.SetNodeAdminState("node-b", "maintenance"); err != nil {
+		t.Fatalf("SetNodeAdminState: %v", err)
+	}
+	waitFor(t, "b to learn its maintenance state", 5*time.Second, func() bool {
+		return memberIDs(b)["node-b"].State == StateMaintenance
+	})
+
+	//Leave
+	if err := b.LeaveCluster(); err != nil {
+		t.Fatalf("LeaveCluster: %v", err)
+	}
+	if b.InCluster() {
+		t.Errorf("b should be standalone after leaving")
+	}
+	waitFor(t, "a to forget b", 5*time.Second, func() bool {
+		_, ok := memberIDs(a)["node-b"]
+		return !ok
+	})
+	if err := b.LeaveCluster(); err != ErrNotInCluster {
+		t.Errorf("leave twice: %v", err)
+	}
+}
+
+func TestJoinRejectsBadToken(t *testing.T) {
+	a, _ := newTestManager(t, "node-a", true)
+	b, _ := newTestManager(t, "node-b", true)
+	if _, err := a.CreateCluster("Home"); err != nil {
+		t.Fatalf("CreateCluster: %v", err)
+	}
+	token, rec, err := a.NewJoinToken(time.Hour)
+	if err != nil {
+		t.Fatalf("NewJoinToken: %v", err)
+	}
+	if err := a.RevokeJoinToken(rec.ID); err != nil {
+		t.Fatalf("RevokeJoinToken: %v", err)
+	}
+	if _, err := b.JoinCluster(token); err == nil {
+		t.Errorf("revoked token must be rejected")
+	}
+	if b.InCluster() {
+		t.Errorf("b must stay standalone")
+	}
+	if _, err := b.JoinCluster("garbage"); err == nil {
+		t.Errorf("garbage token must be rejected")
+	}
+}
+
+func TestTunnelOnlyNodeJoinsAndIsRelayed(t *testing.T) {
+	a, _ := newTestManager(t, "node-a", true)
+	b, _ := newTestManager(t, "node-b", true)
+	c, _ := newTestManager(t, "node-c", false) //NAT-only
+
+	if _, err := a.CreateCluster("Home"); err != nil {
+		t.Fatalf("CreateCluster: %v", err)
+	}
+	token, _, _ := a.NewJoinToken(time.Hour)
+	if _, err := c.JoinCluster(token); err != nil {
+		t.Fatalf("c join: %v", err)
+	}
+	waitFor(t, "c tunnel to a", 10*time.Second, func() bool { return a.hub.Connected("node-c") })
+
+	st := c.Status()
+	if !st.Local.TunnelConnected || st.Local.TunnelHost != "node-a" || st.Local.Reachable {
+		t.Errorf("c status: %+v", st.Local)
+	}
+	if res := a.ProbeNode("node-c"); !res.OK || res.Route != "tunnel" {
+		t.Errorf("a->c probe: %+v", res)
+	}
+	waitFor(t, "c to see a online", 5*time.Second, func() bool {
+		return memberIDs(c)["node-a"].State == StateOnline
+	})
+
+	//Second reachable node joins and must reach c by relaying through a
+	token2, _, _ := a.NewJoinToken(time.Hour)
+	if _, err := b.JoinCluster(token2); err != nil {
+		t.Fatalf("b join: %v", err)
+	}
+	waitFor(t, "b to learn c's tunnel host", 5*time.Second, func() bool {
+		v, ok := memberIDs(b)["node-c"]
+		return ok && v.TunnelVia == "node-a"
+	})
+	res := b.ProbeNode("node-c")
+	if !res.OK || res.Route != "relay via node-a" {
+		t.Errorf("b->c relay probe: %+v", res)
+	}
+	waitFor(t, "c to see b online", 5*time.Second, func() bool {
+		return memberIDs(c)["node-b"].State == StateOnline
+	})
+	waitFor(t, "b to see c online", 5*time.Second, func() bool {
+		return memberIDs(b)["node-c"].State == StateOnline
+	})
+}
+
+func TestEvictNode(t *testing.T) {
+	a, _ := newTestManager(t, "node-a", true)
+	b, _ := newTestManager(t, "node-b", true)
+	if _, err := a.CreateCluster("Home"); err != nil {
+		t.Fatalf("CreateCluster: %v", err)
+	}
+	token, _, _ := a.NewJoinToken(time.Hour)
+	if _, err := b.JoinCluster(token); err != nil {
+		t.Fatalf("join: %v", err)
+	}
+	waitFor(t, "a sees b", 5*time.Second, func() bool { _, ok := memberIDs(a)["node-b"]; return ok })
+
+	if err := a.RemoveNode("node-a"); err == nil {
+		t.Errorf("removing self must fail")
+	}
+	if err := a.RemoveNode("node-b"); err != nil {
+		t.Fatalf("RemoveNode: %v", err)
+	}
+	if _, ok := memberIDs(a)["node-b"]; ok {
+		t.Errorf("a still lists b")
+	}
+	waitFor(t, "b to notice eviction", 10*time.Second, func() bool { return !b.InCluster() })
+	if err := a.RemoveNode("node-b"); err != ErrNodeNotFound {
+		t.Errorf("remove twice: %v", err)
+	}
+}
+
+func TestResumeAfterRestart(t *testing.T) {
+	dir := t.TempDir()
+	opt := Option{NodeID: "node-r", DBFile: filepath.Join(dir, "cluster.db"), KeyFile: filepath.Join(dir, "node.key"), Version: "test", DefaultName: "R"}
+	m, err := NewManager(opt)
+	if err != nil {
+		t.Fatalf("NewManager: %v", err)
+	}
+	cfg := m.Config()
+	cfg.AdvertiseURL = "https://r.example.com"
+	if err := m.UpdateConfig(cfg); err != nil {
+		t.Fatalf("UpdateConfig: %v", err)
+	}
+	info, err := m.CreateCluster("Persist")
+	if err != nil {
+		t.Fatalf("CreateCluster: %v", err)
+	}
+	pub := m.Status().Local.PublicKey
+	m.Close()
+
+	m2, err := NewManager(opt)
+	if err != nil {
+		t.Fatalf("NewManager again: %v", err)
+	}
+	defer m2.Close()
+	if !m2.InCluster() || m2.Cluster().ID != info.ID {
+		t.Errorf("cluster not resumed")
+	}
+	if m2.Config().AdvertiseURL != "https://r.example.com" {
+		t.Errorf("config not persisted")
+	}
+	if m2.Status().Local.PublicKey != pub {
+		t.Errorf("node key changed across restart")
+	}
+	if len(m2.NodeViews()) != 1 || !m2.NodeViews()[0].Local {
+		t.Errorf("local record missing after restart: %+v", m2.NodeViews())
+	}
+}
+
+func TestUpdateConfigValidation(t *testing.T) {
+	m, _ := newTestManager(t, "node-v", true)
+	cfg := m.Config()
+	cfg.Name = ""
+	if err := m.UpdateConfig(cfg); err == nil {
+		t.Errorf("empty name accepted")
+	}
+	cfg = m.Config()
+	cfg.AdvertiseURL = "ftp://nope"
+	if err := m.UpdateConfig(cfg); err == nil {
+		t.Errorf("bad URL scheme accepted")
+	}
+	cfg = m.Config()
+	cfg.TunnelVia = "node-v"
+	if err := m.UpdateConfig(cfg); err != nil || m.Config().TunnelVia != "" {
+		t.Errorf("tunnel via self should be cleared, got %q (%v)", m.Config().TunnelVia, err)
+	}
+}

src/mod/cluster/membership/store.go

@@ -0,0 +1,147 @@
+package membership
+
+/*
+	ArozOS Cluster - persistence
+
+	All cluster state lives in its own key-value database file (cluster.db,
+	separate from the main ao.db) so it can be wiped or moved independently.
+
+	Tables:
+		cluster    "info"      -> ClusterInfo
+		nodes      <nodeID>    -> NodeRecord
+		jointokens <tokenID>   -> JoinToken (with secret hash)
+		config     "local"     -> LocalConfig
+*/
+
+import (
+	"encoding/json"
+
+	"imuslab.com/arozos/mod/database"
+)
+
+const (
+	tableCluster    = "cluster"
+	tableNodes      = "nodes"
+	tableJoinTokens = "jointokens"
+	tableConfig     = "config"
+)
+
+// store wraps the key-value database with typed accessors.
+type store struct {
+	db *database.Database
+}
+
+// storedJoinToken is the on-disk form, which unlike the API form keeps the hash.
+type storedJoinToken struct {
+	ID         string `json:"id"`
+	SecretHash string `json:"secretHash"`
+	Created    int64  `json:"created"`
+	Expires    int64  `json:"expires"`
+	Uses       int    `json:"uses"`
+}
+
+func newStore(dbfile string) (*store, error) {
+	db, err := database.NewDatabase(dbfile, false)
+	if err != nil {
+		return nil, err
+	}
+	for _, table := range []string{tableCluster, tableNodes, tableJoinTokens, tableConfig} {
+		if err := db.NewTable(table); err != nil {
+			db.Close()
+			return nil, err
+		}
+	}
+	return &store{db: db}, nil
+}
+
+func (s *store) close() {
+	s.db.Close()
+}
+
+func (s *store) loadCluster() *ClusterInfo {
+	if !s.db.KeyExists(tableCluster, "info") {
+		return nil
+	}
+	var info ClusterInfo
+	if err := s.db.Read(tableCluster, "info", &info); err != nil || info.ID == "" {
+		return nil
+	}
+	return &info
+}
+
+func (s *store) saveCluster(info *ClusterInfo) error {
+	return s.db.Write(tableCluster, "info", info)
+}
+
+func (s *store) loadConfig() LocalConfig {
+	var cfg LocalConfig
+	if s.db.KeyExists(tableConfig, "local") {
+		s.db.Read(tableConfig, "local", &cfg)
+	}
+	return cfg
+}
+
+func (s *store) saveConfig(cfg LocalConfig) error {
+	return s.db.Write(tableConfig, "local", cfg)
+}
+
+func (s *store) loadNodes() map[string]*NodeRecord {
+	nodes := map[string]*NodeRecord{}
+	entries, err := s.db.ListTable(tableNodes)
+	if err != nil {
+		return nodes
+	}
+	for _, kv := range entries {
+		var rec NodeRecord
+		if err := json.Unmarshal(kv[1], &rec); err != nil || rec.ID == "" {
+			continue
+		}
+		nodes[rec.ID] = &rec
+	}
+	return nodes
+}
+
+func (s *store) saveNode(rec *NodeRecord) error {
+	return s.db.Write(tableNodes, rec.ID, rec)
+}
+
+func (s *store) deleteNode(id string) error {
+	return s.db.Delete(tableNodes, id)
+}
+
+func (s *store) loadJoinTokens() map[string]*JoinToken {
+	tokens := map[string]*JoinToken{}
+	entries, err := s.db.ListTable(tableJoinTokens)
+	if err != nil {
+		return tokens
+	}
+	for _, kv := range entries {
+		var st storedJoinToken
+		if err := json.Unmarshal(kv[1], &st); err != nil || st.ID == "" {
+			continue
+		}
+		tokens[st.ID] = &JoinToken{ID: st.ID, SecretHash: st.SecretHash, Created: st.Created, Expires: st.Expires, Uses: st.Uses}
+	}
+	return tokens
+}
+
+func (s *store) saveJoinToken(t *JoinToken) error {
+	return s.db.Write(tableJoinTokens, t.ID, storedJoinToken{ID: t.ID, SecretHash: t.SecretHash, Created: t.Created, Expires: t.Expires, Uses: t.Uses})
+}
+
+func (s *store) deleteJoinToken(id string) error {
+	return s.db.Delete(tableJoinTokens, id)
+}
+
+// wipeCluster removes every cluster-scoped record but keeps the local config.
+func (s *store) wipeCluster() error {
+	for _, table := range []string{tableCluster, tableNodes, tableJoinTokens} {
+		if err := s.db.DropTable(table); err != nil {
+			return err
+		}
+		if err := s.db.NewTable(table); err != nil {
+			return err
+		}
+	}
+	return nil
+}

src/mod/cluster/membership/types.go

@@ -0,0 +1,208 @@
+package membership
+
+/*
+	ArozOS Cluster - membership types
+
+	A cluster is a set of NodeRecords sharing one ClusterInfo. Records are
+	replicated between nodes by gossip (heartbeats and sync pushes) and merged
+	with last-writer-wins on the Updated timestamp, so every node converges on
+	the same membership view without a permanent master.
+*/
+
+import (
+	"time"
+
+	"imuslab.com/arozos/mod/cluster/capability"
+)
+
+// NodeState is the computed liveness state of a node.
+type NodeState string
+
+const (
+	StateOnline      NodeState = "ONLINE"
+	StateDegraded    NodeState = "DEGRADED"
+	StateOffline     NodeState = "OFFLINE"
+	StateUnknown     NodeState = "UNKNOWN"
+	StateDraining    NodeState = "DRAINING"
+	StateMaintenance NodeState = "MAINTENANCE"
+)
+
+// Admin-selected modes that override the computed state.
+const (
+	AdminStateNormal      = ""
+	AdminStateMaintenance = "maintenance"
+	AdminStateDraining    = "draining"
+)
+
+const (
+	// HeartbeatInterval is how often each node reports to every peer.
+	HeartbeatInterval = 15 * time.Second
+	// OnlineWindow is the maximum silence before a node stops being ONLINE.
+	OnlineWindow = 45 * time.Second
+	// OfflineWindow is the silence after which a node is OFFLINE (UNKNOWN in between).
+	OfflineWindow = 3 * time.Minute
+	// TombstoneTTL is how long a removed node's record is kept for gossip.
+	TombstoneTTL = 7 * 24 * time.Hour
+)
+
+// Health is the transient load snapshot a node ships with each heartbeat.
+type Health struct {
+	CPUUsage  float64 `json:"cpuUsage"` //percent
+	RAMUsed   int64   `json:"ramUsed"`
+	RAMTotal  int64   `json:"ramTotal"`
+	DiskFree  int64   `json:"diskFree"`
+	DiskTotal int64   `json:"diskTotal"`
+	Uptime    int64   `json:"uptime"` //seconds since ArozOS started
+	Timestamp int64   `json:"timestamp"`
+}
+
+// Degraded reports whether the snapshot indicates an overloaded node.
+func (h Health) Degraded() bool {
+	if h.Timestamp == 0 {
+		return false
+	}
+	if h.CPUUsage >= 97 {
+		return true
+	}
+	if h.RAMTotal > 0 && float64(h.RAMUsed)/float64(h.RAMTotal) >= 0.97 {
+		return true
+	}
+	if h.DiskTotal > 0 && float64(h.DiskFree)/float64(h.DiskTotal) <= 0.02 {
+		return true
+	}
+	return false
+}
+
+// NodeRecord is the replicated description of one cluster member.
+type NodeRecord struct {
+	ID           string              `json:"id"`
+	Name         string              `json:"name"`
+	PublicKey    string              `json:"publicKey"`
+	AdvertiseURL string              `json:"advertiseUrl"` //empty for NAT-only nodes
+	TunnelVia    string              `json:"tunnelVia"`    //node terminating this node's tunnel
+	Version      string              `json:"version"`
+	Capabilities capability.Manifest `json:"capabilities"`
+	AdminState   string              `json:"adminState"`
+	Removed      bool                `json:"removed"`
+	Joined       int64               `json:"joined"`
+	Updated      int64               `json:"updated"` //version stamp for last-writer-wins merge
+
+	//Transient fields, merged by recency rather than by Updated
+	LastSeen int64  `json:"lastSeen"`
+	Health   Health `json:"health"`
+}
+
+// Clone returns a deep enough copy for handing out of the manager.
+func (n *NodeRecord) Clone() *NodeRecord {
+	c := *n
+	if n.Capabilities.Features != nil {
+		c.Capabilities.Features = map[string]bool{}
+		for k, v := range n.Capabilities.Features {
+			c.Capabilities.Features[k] = v
+		}
+	}
+	return &c
+}
+
+// ComputeState derives the liveness state of a record at time now. Local is
+// true for the record describing this very node.
+func (n *NodeRecord) ComputeState(now time.Time, local bool) NodeState {
+	switch n.AdminState {
+	case AdminStateMaintenance:
+		return StateMaintenance
+	case AdminStateDraining:
+		return StateDraining
+	}
+	if local {
+		return StateOnline
+	}
+	if n.LastSeen == 0 {
+		return StateUnknown
+	}
+	silence := now.Sub(time.Unix(n.LastSeen, 0))
+	if silence <= OnlineWindow {
+		if n.Health.Degraded() {
+			return StateDegraded
+		}
+		return StateOnline
+	}
+	if silence <= OfflineWindow {
+		return StateUnknown
+	}
+	return StateOffline
+}
+
+// NodeView is a record plus its computed state, as exposed to UIs and APIs.
+type NodeView struct {
+	NodeRecord
+	State  NodeState `json:"state"`
+	Local  bool      `json:"local"`
+	Tunnel bool      `json:"tunnel"` //true when this node currently terminates the peer's tunnel
+}
+
+// ClusterInfo identifies the cluster itself.
+type ClusterInfo struct {
+	ID      string `json:"id"`
+	Name    string `json:"name"`
+	Created int64  `json:"created"`
+}
+
+// LocalConfig is the operator-set configuration of this node.
+type LocalConfig struct {
+	Name         string `json:"name"`
+	AdvertiseURL string `json:"advertiseUrl"`
+	TunnelVia    string `json:"tunnelVia"` //preferred tunnel host, empty for automatic
+	InsecureTLS  bool   `json:"insecureTls"`
+}
+
+// JoinToken is the server-side record of an issued join token.
+type JoinToken struct {
+	ID         string `json:"id"`
+	SecretHash string `json:"-"`
+	Created    int64  `json:"created"`
+	Expires    int64  `json:"expires"`
+	Uses       int    `json:"uses"`
+}
+
+// Wire payloads
+
+type JoinRequest struct {
+	ClusterID string     `json:"clusterId"`
+	TokenID   string     `json:"tokenId"`
+	Secret    string     `json:"secret"`
+	Node      NodeRecord `json:"node"`
+}
+
+type JoinResponse struct {
+	Cluster ClusterInfo  `json:"cluster"`
+	Nodes   []NodeRecord `json:"nodes"`
+	Via     string       `json:"via"` //ID of the node that accepted the join
+}
+
+type HeartbeatRequest struct {
+	Node NodeRecord `json:"node"`
+}
+
+type HeartbeatResponse struct {
+	Nodes []NodeRecord `json:"nodes"`
+	Time  int64        `json:"time"`
+}
+
+type SyncRequest struct {
+	Nodes []NodeRecord `json:"nodes"`
+}
+
+type NodeIDRequest struct {
+	NodeID string `json:"nodeId"`
+}
+
+// nextVersion returns a record version stamp that is strictly greater than
+// prev, so a change made in the same millisecond as the previous one still
+// wins the last-writer-wins merge on every other node.
+func nextVersion(prev int64) int64 {
+	v := time.Now().UnixMilli()
+	if v <= prev {
+		v = prev + 1
+	}
+	return v
+}

src/subservice.go

@@ -28,6 +28,7 @@ var (
 		"STDERR",
 		"COM",
 		"ws",
+		"cluster",
 	}
 )
 

src/web/SystemAO/cluster/cluster.html

@@ -0,0 +1,623 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta name="mobile-web-app-capable" content="yes">
+    <meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1"/>
+    <meta charset="UTF-8">
+    <title>Cluster</title>
+    <script src="../../script/jquery.min.js"></script>
+    <script src="../../script/ao_module.js"></script>
+    <script src="../../script/applocale.js"></script>
+    <style>
+        * { box-sizing: border-box; margin: 0; padding: 0; }
+
+        #cl-root {
+            --cl-bg:        #f5f5f7;
+            --cl-card:      #ffffff;
+            --cl-card-head: #fafafa;
+            --cl-border:    rgba(0,0,0,0.08);
+            --cl-text:      #1d1d1f;
+            --cl-dim:       #6e6e73;
+            --cl-muted:     #98989d;
+            --cl-accent:    #0071e3;
+            --cl-success:   #34c759;
+            --cl-warn:      #ff9f0a;
+            --cl-danger:    #ff3b30;
+            --cl-hover:     rgba(0,0,0,0.04);
+            --cl-input:     #ffffff;
+            --cl-chip:      rgba(0,113,227,0.10);
+            --cl-shadow:    0 1px 4px rgba(0,0,0,0.07), 0 0 0 0.5px rgba(0,0,0,0.06);
+
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif;
+            font-size: 14px;
+            -webkit-font-smoothing: antialiased;
+            color: var(--cl-text);
+            background: var(--cl-bg);
+            padding: 20px;
+            min-height: 100vh;
+        }
+        #cl-root.dark {
+            --cl-bg:        #1f1f1f;
+            --cl-card:      #2c2c2e;
+            --cl-card-head: #232325;
+            --cl-border:    rgba(255,255,255,0.09);
+            --cl-text:      #f2f2f7;
+            --cl-dim:       #aeaeb2;
+            --cl-muted:     #636366;
+            --cl-accent:    #2997ff;
+            --cl-success:   #30d158;
+            --cl-warn:      #ffd60a;
+            --cl-danger:    #ff453a;
+            --cl-hover:     rgba(255,255,255,0.05);
+            --cl-input:     #1c1c1e;
+            --cl-chip:      rgba(41,151,255,0.18);
+            --cl-shadow:    0 1px 6px rgba(0,0,0,0.35), 0 0 0 0.5px rgba(255,255,255,0.06);
+        }
+
+        #cl-header { display: flex; align-items: center; gap: 12px; margin-bottom: 22px; }
+        #cl-header img { width: 40px; height: 40px; border-radius: 9px; }
+        #cl-header h1 { font-size: 20px; font-weight: 600; letter-spacing: -0.2px; }
+        #cl-header p  { font-size: 13px; color: var(--cl-dim); margin-top: 2px; }
+
+        .cl-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
+        @media (max-width: 900px) { .cl-grid { grid-template-columns: 1fr; } }
+
+        .cl-card {
+            background: var(--cl-card); border-radius: 12px; box-shadow: var(--cl-shadow);
+            overflow: hidden; margin-bottom: 16px;
+        }
+        .cl-card-head {
+            display: flex; align-items: center; justify-content: space-between;
+            padding: 12px 16px; background: var(--cl-card-head); border-bottom: 1px solid var(--cl-border);
+            font-weight: 600; font-size: 14px;
+        }
+        .cl-card-head .cl-sub { font-weight: 400; color: var(--cl-dim); font-size: 12px; margin-left: 8px; }
+        .cl-card-body { padding: 16px; }
+
+        .cl-field { margin-bottom: 12px; }
+        .cl-field label { display: block; font-size: 12px; color: var(--cl-dim); margin-bottom: 4px; }
+        .cl-field input[type=text], .cl-field textarea, .cl-field select {
+            width: 100%; padding: 8px 10px; border-radius: 8px; border: 1px solid var(--cl-border);
+            background: var(--cl-input); color: var(--cl-text); font: inherit; font-size: 13px;
+        }
+        .cl-field textarea { min-height: 72px; resize: vertical; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 12px; }
+        .cl-field .cl-hint { font-size: 11px; color: var(--cl-muted); margin-top: 4px; }
+        .cl-row { display: flex; gap: 8px; align-items: center; }
+        .cl-row > * { flex: 1; }
+        .cl-row > .cl-fixed { flex: 0 0 auto; }
+        .cl-check { display: flex; align-items: center; gap: 8px; font-size: 13px; color: var(--cl-text); cursor: pointer; }
+
+        .cl-btn {
+            display: inline-flex; align-items: center; gap: 6px; padding: 7px 14px; border-radius: 8px;
+            border: 1px solid var(--cl-border); background: var(--cl-card); color: var(--cl-text);
+            font: inherit; font-size: 13px; cursor: pointer; white-space: nowrap;
+        }
+        .cl-btn:hover { background: var(--cl-hover); }
+        .cl-btn.primary { background: var(--cl-accent); border-color: transparent; color: #fff; }
+        .cl-btn.primary:hover { filter: brightness(1.08); }
+        .cl-btn.danger { color: var(--cl-danger); }
+        .cl-btn.small { padding: 4px 9px; font-size: 12px; }
+        .cl-btn:disabled { opacity: 0.5; cursor: default; }
+        .cl-btn svg { width: 13px; height: 13px; }
+
+        .cl-kv { display: grid; grid-template-columns: 130px 1fr; gap: 6px 12px; font-size: 13px; }
+        .cl-kv dt { color: var(--cl-dim); }
+        .cl-kv dd { word-break: break-all; }
+        .cl-mono { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 12px; }
+
+        .cl-chips { display: flex; flex-wrap: wrap; gap: 6px; }
+        .cl-chip { background: var(--cl-chip); color: var(--cl-accent); border-radius: 999px; padding: 2px 9px; font-size: 11px; font-weight: 500; }
+
+        .cl-badge { display: inline-block; padding: 2px 8px; border-radius: 999px; font-size: 11px; font-weight: 600; letter-spacing: 0.2px; color: #fff; }
+        .cl-badge.ONLINE      { background: var(--cl-success); }
+        .cl-badge.DEGRADED    { background: var(--cl-warn); color: #1d1d1f; }
+        .cl-badge.UNKNOWN     { background: var(--cl-muted); }
+        .cl-badge.OFFLINE     { background: var(--cl-danger); }
+        .cl-badge.MAINTENANCE { background: #5e5ce6; }
+        .cl-badge.DRAINING    { background: #ff9f0a; color: #1d1d1f; }
+
+        .cl-bar { height: 6px; border-radius: 3px; background: var(--cl-border); overflow: hidden; margin-top: 4px; }
+        .cl-bar > div { height: 100%; background: var(--cl-accent); }
+        .cl-bar.warn > div { background: var(--cl-warn); }
+        .cl-bar.bad > div { background: var(--cl-danger); }
+
+        table.cl-table { width: 100%; border-collapse: collapse; font-size: 13px; }
+        table.cl-table th { text-align: left; font-weight: 600; color: var(--cl-dim); font-size: 12px; padding: 8px 10px; border-bottom: 1px solid var(--cl-border); }
+        table.cl-table td { padding: 10px; border-bottom: 1px solid var(--cl-border); vertical-align: top; }
+        table.cl-table tr:last-child td { border-bottom: none; }
+        table.cl-table tr:hover td { background: var(--cl-hover); }
+        .cl-node-name { font-weight: 600; }
+        .cl-node-meta { color: var(--cl-dim); font-size: 12px; margin-top: 2px; }
+        .cl-actions { display: flex; gap: 4px; flex-wrap: wrap; }
+
+        .cl-empty { text-align: center; color: var(--cl-muted); padding: 24px 0; font-size: 13px; }
+        .cl-msg { padding: 8px 12px; border-radius: 8px; font-size: 13px; margin-top: 10px; display: none; }
+        .cl-msg.ok { display: block; background: rgba(52,199,89,0.14); color: var(--cl-success); }
+        .cl-msg.err { display: block; background: rgba(255,59,48,0.12); color: var(--cl-danger); }
+        .cl-sep { height: 1px; background: var(--cl-border); margin: 14px 0; }
+        .cl-token-box {
+            background: var(--cl-input); border: 1px dashed var(--cl-border); border-radius: 8px; padding: 10px;
+            font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 11px; word-break: break-all; margin-top: 8px;
+        }
+        .cl-inline-list { list-style: none; }
+        .cl-inline-list li { display: flex; justify-content: space-between; align-items: center; padding: 6px 0; border-bottom: 1px solid var(--cl-border); font-size: 12px; }
+        .cl-inline-list li:last-child { border-bottom: none; }
+        #cl-refresh { color: var(--cl-muted); font-size: 11px; }
+    </style>
+</head>
+<body>
+<div id="cl-root">
+    <div id="cl-header">
+        <img src="img/small_icon.png" alt="">
+        <div style="flex:1">
+            <h1 id="cl-title">Cluster</h1>
+            <p id="cl-tagline">Several ArozOS hosts working as one logical computer</p>
+        </div>
+        <span id="cl-refresh"></span>
+    </div>
+
+    <div class="cl-grid">
+        <!-- This node -->
+        <div>
+            <div class="cl-card">
+                <div class="cl-card-head">
+                    <span><span id="lbl-thisnode">This node</span><span class="cl-sub cl-mono" id="cl-nodeid"></span></span>
+                    <button class="cl-btn small" onclick="refreshCapabilities()" title="Re-detect capabilities">
+                        <svg viewBox="0 0 16 16" fill="none"><path d="M13 8a5 5 0 1 1-1.5-3.6" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/><path d="M13 2v3h-3" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/></svg>
+                        <span id="lbl-redetect">Re-detect</span>
+                    </button>
+                </div>
+                <div class="cl-card-body">
+                    <div class="cl-field">
+                        <label id="lbl-name">Node name</label>
+                        <input type="text" id="cfg-name" maxlength="64">
+                    </div>
+                    <div class="cl-field">
+                        <label id="lbl-url">Advertised URL (how other nodes reach this one)</label>
+                        <div class="cl-row">
+                            <input type="text" id="cfg-url" placeholder="https://node-a.example.com">
+                            <button class="cl-btn cl-fixed" onclick="testUrl()"><span id="lbl-test">Test</span></button>
+                        </div>
+                        <div class="cl-hint" id="hint-url">Leave empty if this node has no public address. It will then keep a tunnel open to a reachable member, which also works through Cloudflare.</div>
+                    </div>
+                    <div class="cl-field" id="field-tunnel" style="display:none">
+                        <label id="lbl-tunnel">Tunnel through</label>
+                        <select id="cfg-tunnelvia"><option value="">Automatic (any reachable member)</option></select>
+                    </div>
+                    <div class="cl-field">
+                        <label class="cl-check"><input type="checkbox" id="cfg-insecure"> <span id="lbl-insecure">Accept self-signed certificates from other nodes (LAN only)</span></label>
+                    </div>
+                    <div class="cl-row">
+                        <div></div>
+                        <button class="cl-btn primary cl-fixed" onclick="saveConfig()"><span id="lbl-save">Save</span></button>
+                    </div>
+                    <div class="cl-msg" id="msg-config"></div>
+
+                    <div class="cl-sep"></div>
+                    <dl class="cl-kv">
+                        <dt id="lbl-reach">Reachability</dt><dd id="v-reach">-</dd>
+                        <dt id="lbl-version">Version</dt><dd id="v-version">-</dd>
+                        <dt id="lbl-platform">Platform</dt><dd id="v-platform">-</dd>
+                        <dt id="lbl-key">Node key</dt><dd class="cl-mono" id="v-key">-</dd>
+                        <dt id="lbl-caps">Capabilities</dt><dd><div class="cl-chips" id="v-caps"></div></dd>
+                    </dl>
+                    <div class="cl-sep"></div>
+                    <dl class="cl-kv" id="v-health"></dl>
+                </div>
+            </div>
+        </div>
+
+        <!-- Cluster -->
+        <div>
+            <div class="cl-card" id="card-standalone" style="display:none">
+                <div class="cl-card-head"><span id="lbl-notmember">Not part of a cluster</span></div>
+                <div class="cl-card-body">
+                    <div class="cl-field">
+                        <label id="lbl-create">Create a new cluster</label>
+                        <div class="cl-row">
+                            <input type="text" id="create-name" placeholder="Home Lab" maxlength="64">
+                            <button class="cl-btn primary cl-fixed" onclick="createCluster()"><span id="lbl-createbtn">Create</span></button>
+                        </div>
+                        <div class="cl-hint" id="hint-create">This node becomes the first member. Generate join tokens here afterwards to add more nodes.</div>
+                    </div>
+                    <div class="cl-sep"></div>
+                    <div class="cl-field">
+                        <label id="lbl-join">Join an existing cluster</label>
+                        <textarea id="join-token" placeholder="aroz-join:..."></textarea>
+                        <div class="cl-hint" id="hint-join">Paste a join token generated on a reachable member of the cluster.</div>
+                    </div>
+                    <div class="cl-row">
+                        <div></div>
+                        <button class="cl-btn primary cl-fixed" onclick="joinCluster()"><span id="lbl-joinbtn">Join</span></button>
+                    </div>
+                    <div class="cl-msg" id="msg-cluster"></div>
+                </div>
+            </div>
+
+            <div class="cl-card" id="card-member" style="display:none">
+                <div class="cl-card-head">
+                    <span id="v-clustername">Cluster</span>
+                    <button class="cl-btn small danger" onclick="leaveCluster()"><span id="lbl-leave">Leave cluster</span></button>
+                </div>
+                <div class="cl-card-body">
+                    <dl class="cl-kv">
+                        <dt id="lbl-clusterid">Cluster ID</dt><dd class="cl-mono" id="v-clusterid">-</dd>
+                        <dt id="lbl-created">Created</dt><dd id="v-created">-</dd>
+                        <dt id="lbl-members">Members</dt><dd id="v-membercount">-</dd>
+                    </dl>
+                    <div class="cl-sep"></div>
+                    <div class="cl-field">
+                        <label id="lbl-invite">Add a node</label>
+                        <div class="cl-row">
+                            <select id="token-ttl" class="cl-fixed" style="flex:0 0 auto; width:auto">
+                                <option value="1">1 hour</option>
+                                <option value="24" selected>24 hours</option>
+                                <option value="168">7 days</option>
+                            </select>
+                            <button class="cl-btn primary" onclick="newToken()" id="btn-token"><span id="lbl-gentoken">Generate join token</span></button>
+                        </div>
+                        <div class="cl-hint" id="hint-token">Tokens can only be generated on a node with an advertised URL. Paste the token into the other node's Cluster settings.</div>
+                        <div id="token-out" style="display:none">
+                            <div class="cl-token-box" id="token-text"></div>
+                            <div class="cl-row" style="margin-top:6px">
+                                <div class="cl-hint" id="token-exp"></div>
+                                <button class="cl-btn small cl-fixed" onclick="copyToken()"><span id="lbl-copy">Copy</span></button>
+                            </div>
+                        </div>
+                    </div>
+                    <ul class="cl-inline-list" id="token-list"></ul>
+                    <div class="cl-msg" id="msg-member"></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- Members -->
+    <div class="cl-card" id="card-nodes" style="display:none">
+        <div class="cl-card-head"><span id="lbl-nodes">Nodes</span></div>
+        <div class="cl-card-body" style="padding:0; overflow-x:auto">
+            <table class="cl-table">
+                <thead>
+                    <tr>
+                        <th id="th-node">Node</th>
+                        <th id="th-state">State</th>
+                        <th id="th-route">Route</th>
+                        <th id="th-platform">Platform</th>
+                        <th id="th-load">Load</th>
+                        <th id="th-seen">Last seen</th>
+                        <th></th>
+                    </tr>
+                </thead>
+                <tbody id="nodes-body"></tbody>
+            </table>
+        </div>
+    </div>
+</div>
+
+<script>
+    /* i18n */
+    var clLocale = (typeof NewAppLocale === 'function') ? NewAppLocale() : null;
+    function t(key, fallback) { return clLocale ? clLocale.getString(key, fallback) : fallback; }
+
+    /* Theme */
+    function clApplyTheme(isDark) { document.getElementById('cl-root').classList.toggle('dark', isDark); }
+    (function() {
+        try {
+            if (typeof ao_module_getSystemThemeColor === 'function') {
+                ao_module_getSystemThemeColor(function(c) { clApplyTheme(c !== 'whiteTheme'); });
+                return;
+            }
+        } catch(e) {}
+        try {
+            var theme = null;
+            if (typeof preferredTheme !== 'undefined') theme = preferredTheme;
+            else if (parent && typeof parent.preferredTheme !== 'undefined') theme = parent.preferredTheme;
+            if (theme) clApplyTheme(theme === 'dark' || theme === 'darkTheme');
+        } catch(e) {}
+    })();
+    window.detailPageThemeCallback = function(isDark) { clApplyTheme(isDark); };
+
+    /* Helpers */
+    var API = '../../system/cluster/';
+    var clusterStatus = null;
+    var configDirty = false;
+
+    function esc(s) {
+        return String(s === undefined || s === null ? '' : s)
+            .replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');
+    }
+    function fmtBytes(b) {
+        if (!b || b <= 0) return '-';
+        var u = ['B','KB','MB','GB','TB','PB'], i = 0;
+        while (b >= 1024 && i < u.length - 1) { b /= 1024; i++; }
+        return (i === 0 ? b : b.toFixed(1)) + ' ' + u[i];
+    }
+    function fmtAgo(unix) {
+        if (!unix) return t('cluster/never', 'never');
+        var d = Math.max(0, Math.floor(Date.now()/1000 - unix));
+        if (d < 5) return t('cluster/justnow', 'just now');
+        if (d < 60) return d + 's ' + t('cluster/ago', 'ago');
+        if (d < 3600) return Math.floor(d/60) + 'm ' + t('cluster/ago', 'ago');
+        if (d < 86400) return Math.floor(d/3600) + 'h ' + t('cluster/ago', 'ago');
+        return Math.floor(d/86400) + 'd ' + t('cluster/ago', 'ago');
+    }
+    function fmtDate(unix) { return unix ? new Date(unix * 1000).toLocaleString() : '-'; }
+    function showMsg(id, text, ok) {
+        var el = document.getElementById(id);
+        el.textContent = text;
+        el.className = 'cl-msg ' + (ok ? 'ok' : 'err');
+        clearTimeout(el._t);
+        el._t = setTimeout(function() { el.className = 'cl-msg'; }, ok ? 4000 : 8000);
+    }
+    function apiPost(path, data, cb) {
+        $.ajax({ url: API + path, method: 'POST', data: data, dataType: 'json' })
+            .done(function(r) { cb(null, r); })
+            .fail(function(xhr) {
+                var msg = xhr.responseText || 'request failed';
+                try { var j = JSON.parse(xhr.responseText); if (j.error) msg = j.error; } catch(e) {}
+                cb(msg, null);
+            });
+    }
+    function apiResult(r) { return (r && typeof r === 'object' && r.error) ? r.error : null; }
+
+    /* Rendering */
+    function bar(pct) {
+        var cls = pct >= 95 ? 'bad' : (pct >= 80 ? 'warn' : '');
+        return '<div class="cl-bar ' + cls + '"><div style="width:' + Math.min(100, Math.max(0, pct)).toFixed(0) + '%"></div></div>';
+    }
+    function healthRows(h) {
+        if (!h || !h.timestamp) return '<dt>' + t('cluster/load', 'Load') + '</dt><dd style="color:var(--cl-muted)">' + t('cluster/noreport', 'no report yet') + '</dd>';
+        var ramPct = h.ramTotal > 0 ? h.ramUsed / h.ramTotal * 100 : 0;
+        var diskPct = h.diskTotal > 0 ? (h.diskTotal - h.diskFree) / h.diskTotal * 100 : 0;
+        return '<dt>CPU</dt><dd>' + h.cpuUsage.toFixed(0) + '%' + bar(h.cpuUsage) + '</dd>' +
+               '<dt>RAM</dt><dd>' + (h.ramTotal > 0 ? fmtBytes(h.ramUsed) + ' / ' + fmtBytes(h.ramTotal) + bar(ramPct) : '-') + '</dd>' +
+               '<dt>' + t('cluster/disk', 'Disk') + '</dt><dd>' + (h.diskTotal > 0 ? fmtBytes(h.diskFree) + ' ' + t('cluster/free', 'free') + ' / ' + fmtBytes(h.diskTotal) + bar(diskPct) : '-') + '</dd>' +
+               '<dt>' + t('cluster/uptime', 'Uptime') + '</dt><dd>' + Math.floor(h.uptime / 3600) + 'h ' + Math.floor((h.uptime % 3600) / 60) + 'm</dd>';
+    }
+    function routeOf(n) {
+        if (n.local) return t('cluster/thisnode', 'This node');
+        if (n.tunnel) return t('cluster/tunnelhere', 'Tunnel to this node');
+        if (n.advertiseUrl) return '<a href="' + esc(n.advertiseUrl) + '" target="_blank" style="color:var(--cl-accent)">' + esc(n.advertiseUrl.replace(/^https?:\/\//, '')) + '</a>';
+        if (n.tunnelVia) return t('cluster/tunnelvia', 'Tunnel via') + ' ' + esc(nodeName(n.tunnelVia));
+        return '<span style="color:var(--cl-muted)">' + t('cluster/noroute', 'No route') + '</span>';
+    }
+    function nodeName(id) {
+        if (!clusterStatus) return id;
+        for (var i = 0; i < clusterStatus.nodes.length; i++) if (clusterStatus.nodes[i].id === id) return clusterStatus.nodes[i].name;
+        return id.substring(0, 8);
+    }
+
+    function render() {
+        var s = clusterStatus;
+        var local = s.local;
+        document.getElementById('cl-nodeid').textContent = ' ' + local.nodeId;
+        if (!configDirty) {
+            document.getElementById('cfg-name').value = local.config.name || '';
+            document.getElementById('cfg-url').value = local.config.advertiseUrl || '';
+            document.getElementById('cfg-insecure').checked = !!local.config.insecureTls;
+        }
+        renderTunnelSelect(local.config.tunnelVia || '');
+        toggleTunnelField();
+
+        var reach;
+        if (local.reachable) {
+            reach = '<span class="cl-badge ONLINE">' + t('cluster/direct', 'DIRECT') + '</span> ' + esc(local.config.advertiseUrl);
+        } else if (local.tunnelConnected) {
+            reach = '<span class="cl-badge ONLINE">' + t('cluster/tunnel', 'TUNNEL') + '</span> ' + t('cluster/via', 'via') + ' ' + esc(nodeName(local.tunnelHost));
+        } else if (s.inCluster) {
+            reach = '<span class="cl-badge OFFLINE">' + t('cluster/notunnel', 'NO TUNNEL') + '</span> ' + t('cluster/waitinghost', 'waiting for a reachable member');
+        } else {
+            reach = '<span class="cl-badge UNKNOWN">' + t('cluster/natonly', 'NO URL') + '</span> ' + t('cluster/willtunnel', 'will tunnel through a member once in a cluster');
+        }
+        document.getElementById('v-reach').innerHTML = reach;
+        document.getElementById('v-version').textContent = local.version;
+        var c = local.capabilities || {};
+        document.getElementById('v-platform').textContent = (c.os || '?') + ' / ' + (c.arch || '?') + ', ' + (c.cpuCores || '?') + ' ' + t('cluster/cores', 'cores') + (c.totalRam > 0 ? ', ' + fmtBytes(c.totalRam) + ' RAM' : '') + (c.hostname ? ' (' + c.hostname + ')' : '');
+        document.getElementById('v-key').textContent = (local.publicKey || '').substring(0, 20) + '...';
+        var chips = '';
+        var feats = Object.keys(c.features || {}).filter(function(k) { return c.features[k]; }).sort();
+        feats.forEach(function(f) { chips += '<span class="cl-chip">' + esc(f) + '</span>'; });
+        document.getElementById('v-caps').innerHTML = chips || '<span style="color:var(--cl-muted)">-</span>';
+        document.getElementById('v-health').innerHTML = healthRows(local.health);
+
+        document.getElementById('card-standalone').style.display = s.inCluster ? 'none' : '';
+        document.getElementById('card-member').style.display = s.inCluster ? '' : 'none';
+        document.getElementById('card-nodes').style.display = s.inCluster ? '' : 'none';
+
+        if (s.inCluster) {
+            document.getElementById('v-clustername').textContent = s.cluster.name;
+            document.getElementById('v-clusterid').textContent = s.cluster.id;
+            document.getElementById('v-created').textContent = fmtDate(s.cluster.created);
+            document.getElementById('v-membercount').textContent = s.nodes.length;
+            document.getElementById('btn-token').disabled = !local.reachable;
+            renderTokens(s.tokens || []);
+            renderNodes(s.nodes || []);
+        }
+        document.getElementById('cl-refresh').textContent = t('cluster/updated', 'Updated') + ' ' + new Date().toLocaleTimeString();
+    }
+
+    function renderTunnelSelect(current) {
+        var sel = document.getElementById('cfg-tunnelvia');
+        var prev = configDirty ? sel.value : current;
+        var html = '<option value="">' + t('cluster/auto', 'Automatic (any reachable member)') + '</option>';
+        (clusterStatus.nodes || []).forEach(function(n) {
+            if (n.local || !n.advertiseUrl) return;
+            html += '<option value="' + esc(n.id) + '"' + (n.id === prev ? ' selected' : '') + '>' + esc(n.name) + ' (' + esc(n.advertiseUrl) + ')</option>';
+        });
+        sel.innerHTML = html;
+        sel.value = prev;
+    }
+    function toggleTunnelField() {
+        var url = document.getElementById('cfg-url').value.trim();
+        document.getElementById('field-tunnel').style.display = (url === '' && clusterStatus && clusterStatus.inCluster) ? '' : 'none';
+    }
+
+    function renderTokens(tokens) {
+        var html = '';
+        tokens.forEach(function(tk) {
+            html += '<li><span class="cl-mono">' + esc(tk.id) + '</span><span style="color:var(--cl-dim)">' +
+                    t('cluster/expires', 'expires') + ' ' + fmtDate(tk.expires) + ' · ' + tk.uses + ' ' + t('cluster/uses', 'uses') + '</span>' +
+                    '<button class="cl-btn small danger" onclick="revokeToken(\'' + esc(tk.id) + '\')">' + t('cluster/revoke', 'Revoke') + '</button></li>';
+        });
+        document.getElementById('token-list').innerHTML = html;
+    }
+
+    function renderNodes(nodes) {
+        var html = '';
+        nodes.forEach(function(n) {
+            var c = n.capabilities || {};
+            var h = n.health || {};
+            var load = h.timestamp
+                ? 'CPU ' + h.cpuUsage.toFixed(0) + '%' + (h.ramTotal > 0 ? ' · RAM ' + (h.ramUsed / h.ramTotal * 100).toFixed(0) + '%' : '') + (h.diskTotal > 0 ? ' · ' + fmtBytes(h.diskFree) + ' ' + t('cluster/free', 'free') : '')
+                : '<span style="color:var(--cl-muted)">-</span>';
+            var feats = Object.keys(c.features || {}).filter(function(k) { return c.features[k] && ['ffmpeg','docker','cuda','gpu','git'].indexOf(k) >= 0; });
+            var actions = '';
+            if (!n.local) {
+                actions += '<button class="cl-btn small" onclick="probeNode(\'' + esc(n.id) + '\', this)">' + t('cluster/probe', 'Probe') + '</button>';
+            }
+            var isMaint = n.adminState === 'maintenance';
+            actions += '<button class="cl-btn small" onclick="setNodeState(\'' + esc(n.id) + '\', \'' + (isMaint ? 'normal' : 'maintenance') + '\')">' + (isMaint ? t('cluster/resume', 'Resume') : t('cluster/maintenance', 'Maintenance')) + '</button>';
+            if (!n.local) {
+                actions += '<button class="cl-btn small danger" onclick="removeNode(\'' + esc(n.id) + '\', \'' + esc(n.name) + '\')">' + t('cluster/remove', 'Remove') + '</button>';
+            }
+            html += '<tr>' +
+                '<td><div class="cl-node-name">' + esc(n.name) + (n.local ? ' <span class="cl-chip">' + t('cluster/you', 'you') + '</span>' : '') + '</div><div class="cl-node-meta cl-mono">' + esc(n.id) + '</div></td>' +
+                '<td><span class="cl-badge ' + esc(n.state) + '">' + esc(n.state) + '</span></td>' +
+                '<td>' + routeOf(n) + '</td>' +
+                '<td>' + esc(c.os || '?') + '/' + esc(c.arch || '?') + '<div class="cl-node-meta">' + (c.cpuCores || '?') + ' ' + t('cluster/cores', 'cores') + (c.totalRam > 0 ? ', ' + fmtBytes(c.totalRam) : '') + (feats.length ? ' · ' + feats.join(', ') : '') + '</div></td>' +
+                '<td>' + load + '</td>' +
+                '<td>' + (n.local ? t('cluster/now', 'now') : fmtAgo(n.lastSeen)) + '<div class="cl-node-meta">' + esc(n.version) + '</div></td>' +
+                '<td><div class="cl-actions">' + actions + '</div></td>' +
+                '</tr>';
+        });
+        document.getElementById('nodes-body').innerHTML = html || '<tr><td colspan="7" class="cl-empty">' + t('cluster/nonodes', 'No nodes') + '</td></tr>';
+    }
+
+    /* Actions */
+    function loadStatus() {
+        $.getJSON(API + 'status', function(data) {
+            if (apiResult(data)) return;
+            clusterStatus = data;
+            render();
+        });
+    }
+    function saveConfig() {
+        apiPost('config', {
+            name: document.getElementById('cfg-name').value,
+            url: document.getElementById('cfg-url').value,
+            tunnelvia: document.getElementById('cfg-tunnelvia').value,
+            insecure: document.getElementById('cfg-insecure').checked
+        }, function(err, r) {
+            var e = err || apiResult(r);
+            if (e) { showMsg('msg-config', e, false); return; }
+            configDirty = false;
+            showMsg('msg-config', t('cluster/saved', 'Saved'), true);
+            loadStatus();
+        });
+    }
+    function testUrl() {
+        var url = document.getElementById('cfg-url').value.trim();
+        if (!url) { showMsg('msg-config', t('cluster/enterurl', 'Enter a URL to test'), false); return; }
+        showMsg('msg-config', t('cluster/testing', 'Testing...'), true);
+        apiPost('testurl', { url: url }, function(err, r) {
+            var e = err || apiResult(r);
+            if (e) { showMsg('msg-config', e, false); return; }
+            showMsg('msg-config', t('cluster/reachableok', 'Reachable, round trip') + ' ' + r.latencyMs + ' ms', true);
+        });
+    }
+    function refreshCapabilities() {
+        $.getJSON(API + 'capabilities?refresh=true', function() { loadStatus(); });
+    }
+    function createCluster() {
+        var name = document.getElementById('create-name').value.trim();
+        if (!name) { showMsg('msg-cluster', t('cluster/entername', 'Enter a cluster name'), false); return; }
+        apiPost('create', { name: name }, function(err, r) {
+            var e = err || apiResult(r);
+            if (e) { showMsg('msg-cluster', e, false); return; }
+            loadStatus();
+        });
+    }
+    function joinCluster() {
+        var token = document.getElementById('join-token').value.trim();
+        if (!token) { showMsg('msg-cluster', t('cluster/entertoken', 'Paste a join token'), false); return; }
+        showMsg('msg-cluster', t('cluster/joining', 'Joining...'), true);
+        apiPost('join', { token: token }, function(err, r) {
+            var e = err || apiResult(r);
+            if (e) { showMsg('msg-cluster', e, false); return; }
+            document.getElementById('join-token').value = '';
+            loadStatus();
+        });
+    }
+    function leaveCluster() {
+        if (!confirm(t('cluster/confirmleave', 'Leave the cluster? Files and jobs on other nodes will no longer be reachable from here.'))) return;
+        apiPost('leave', {}, function(err, r) {
+            var e = err || apiResult(r);
+            if (e) { showMsg('msg-member', e, false); return; }
+            document.getElementById('token-out').style.display = 'none';
+            loadStatus();
+        });
+    }
+    function newToken() {
+        apiPost('token/new', { ttl: document.getElementById('token-ttl').value }, function(err, r) {
+            var e = err || apiResult(r);
+            if (e) { showMsg('msg-member', e, false); return; }
+            document.getElementById('token-text').textContent = r.token;
+            document.getElementById('token-exp').textContent = t('cluster/expires', 'expires') + ' ' + fmtDate(r.expires);
+            document.getElementById('token-out').style.display = '';
+            loadStatus();
+        });
+    }
+    function copyToken() {
+        var text = document.getElementById('token-text').textContent;
+        var done = function() { showMsg('msg-member', t('cluster/copied', 'Token copied to clipboard'), true); };
+        if (navigator.clipboard && navigator.clipboard.writeText) {
+            navigator.clipboard.writeText(text).then(done, function() { fallbackCopy(text); done(); });
+        } else { fallbackCopy(text); done(); }
+    }
+    function fallbackCopy(text) {
+        var ta = document.createElement('textarea');
+        ta.value = text; document.body.appendChild(ta); ta.select();
+        try { document.execCommand('copy'); } catch(e) {}
+        document.body.removeChild(ta);
+    }
+    function revokeToken(id) {
+        apiPost('token/revoke', { id: id }, function(err, r) {
+            var e = err || apiResult(r);
+            if (e) { showMsg('msg-member', e, false); return; }
+            loadStatus();
+        });
+    }
+    function probeNode(id, btn) {
+        btn.disabled = true;
+        $.getJSON(API + 'node/probe?id=' + encodeURIComponent(id), function(r) {
+            btn.disabled = false;
+            if (r.ok) showMsg('msg-member', nodeName(id) + ': ' + r.latencyMs + ' ms (' + r.route + ')', true);
+            else showMsg('msg-member', nodeName(id) + ': ' + (r.error || 'unreachable') + ' (' + r.route + ')', false);
+        }).fail(function() { btn.disabled = false; });
+    }
+    function setNodeState(id, state) {
+        apiPost('node/state', { id: id, state: state }, function(err, r) {
+            var e = err || apiResult(r);
+            if (e) { showMsg('msg-member', e, false); return; }
+            loadStatus();
+        });
+    }
+    function removeNode(id, name) {
+        if (!confirm(t('cluster/confirmremove', 'Remove node') + ' "' + name + '"?')) return;
+        apiPost('node/remove', { id: id }, function(err, r) {
+            var e = err || apiResult(r);
+            if (e) { showMsg('msg-member', e, false); return; }
+            loadStatus();
+        });
+    }
+
+    /* Wire up */
+    ['cfg-name', 'cfg-url', 'cfg-tunnelvia', 'cfg-insecure'].forEach(function(id) {
+        document.getElementById(id).addEventListener('input', function() { configDirty = true; toggleTunnelField(); });
+        document.getElementById(id).addEventListener('change', function() { configDirty = true; toggleTunnelField(); });
+    });
+    loadStatus();
+    setInterval(loadStatus, 10000);
+</script>
+</body>
+</html>