manager.go 31 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229
  1. package membership
  2. /*
  3. ArozOS Cluster - membership manager
  4. The Manager is the cluster agent of one node. It owns the node key, the
  5. cluster database, the ACN server/transport and keeps the replicated
  6. membership view converged with the other nodes through heartbeats.
  7. Life cycle of a node:
  8. standalone --CreateCluster/JoinCluster--> member --LeaveCluster/evicted--> standalone
  9. Every node stays a fully working standalone ArozOS whatever its state.
  10. */
  11. import (
  12. "context"
  13. "crypto/ed25519"
  14. "encoding/json"
  15. "errors"
  16. "net/http"
  17. "sort"
  18. "strings"
  19. "sync"
  20. "time"
  21. uuid "github.com/satori/go.uuid"
  22. "imuslab.com/arozos/mod/cluster/acn"
  23. "imuslab.com/arozos/mod/cluster/capability"
  24. "imuslab.com/arozos/mod/database"
  25. "imuslab.com/arozos/mod/info/logger"
  26. )
  27. // Option configures a Manager.
  28. type Option struct {
  29. NodeID string
  30. DBFile string
  31. KeyFile string
  32. Version string
  33. DefaultName string
  34. Capabilities func() capability.Manifest
  35. Health func() Health
  36. }
  37. // Manager is the cluster agent of this node.
  38. type Manager struct {
  39. opt Option
  40. store *store
  41. key *acn.NodeKey
  42. mu sync.RWMutex
  43. cluster *ClusterInfo
  44. nodes map[string]*NodeRecord
  45. tokens map[string]*JoinToken
  46. config LocalConfig
  47. caps capability.Manifest
  48. signer *acn.Signer
  49. verifier *acn.Verifier
  50. hub *acn.TunnelHub
  51. server *acn.Server
  52. transport *acn.Transport
  53. tunnel *acn.TunnelClient
  54. loopMu sync.Mutex
  55. loopStop chan struct{}
  56. loopWG sync.WaitGroup
  57. reachable map[string]bool //last known reachability per peer, for log de-duplication
  58. latency map[string]float64 //EWMA round trip to each peer, milliseconds
  59. latencyMx latencyCache //cached pairwise matrix collected from peers
  60. started time.Time
  61. //OnClusterChange fires (outside the lock) whenever the replicated
  62. //cluster-wide settings change, locally or through gossip.
  63. OnClusterChange func()
  64. //OnMembershipChange fires (outside the lock) when this node enters or
  65. //leaves a cluster, so sibling services can mount / unmount resources.
  66. OnMembershipChange func(inCluster bool)
  67. }
  68. func (m *Manager) fireMembershipChange(inCluster bool) {
  69. m.mu.RLock()
  70. cb := m.OnMembershipChange
  71. m.mu.RUnlock()
  72. if cb != nil {
  73. cb(inCluster)
  74. }
  75. }
  76. var (
  77. ErrNotInCluster = errors.New("this node is not part of a cluster")
  78. ErrAlreadyInCluster = errors.New("this node is already part of a cluster, leave it first")
  79. ErrNodeNotFound = errors.New("node not found")
  80. )
  81. // NewManager opens the cluster database and node key and, when the node was
  82. // part of a cluster before the restart, resumes membership.
  83. func NewManager(opt Option) (*Manager, error) {
  84. if opt.NodeID == "" {
  85. return nil, errors.New("node id is required")
  86. }
  87. if opt.Capabilities == nil {
  88. opt.Capabilities = capability.Detect
  89. }
  90. if opt.Health == nil {
  91. opt.Health = func() Health { return Health{} }
  92. }
  93. st, err := newStore(opt.DBFile)
  94. if err != nil {
  95. return nil, err
  96. }
  97. key, err := acn.LoadOrCreateNodeKey(opt.KeyFile)
  98. if err != nil {
  99. st.close()
  100. return nil, err
  101. }
  102. m := &Manager{
  103. opt: opt,
  104. store: st,
  105. key: key,
  106. nodes: map[string]*NodeRecord{},
  107. tokens: map[string]*JoinToken{},
  108. reachable: map[string]bool{},
  109. latency: map[string]float64{},
  110. started: time.Now(),
  111. }
  112. m.caps = opt.Capabilities()
  113. m.config = st.loadConfig()
  114. if m.config.Name == "" {
  115. m.config.Name = opt.DefaultName
  116. }
  117. m.cluster = st.loadCluster()
  118. m.nodes = st.loadNodes()
  119. m.tokens = st.loadJoinTokens()
  120. clusterID := ""
  121. if m.cluster != nil {
  122. clusterID = m.cluster.ID
  123. }
  124. m.signer = &acn.Signer{NodeID: opt.NodeID, ClusterID: clusterID, Key: key}
  125. m.verifier = acn.NewVerifier(m.clusterID, m)
  126. m.hub = acn.NewTunnelHub(m.verifier)
  127. m.hub.OnConnect = m.onTunnelConnect
  128. m.server = acn.NewServer(m.verifier, m.hub, opt.Version)
  129. m.registerACNHandlers()
  130. m.transport = acn.NewTransport(m.signer, m, m.hub, m.config.InsecureTLS)
  131. m.tunnel = &acn.TunnelClient{
  132. Signer: m.signer,
  133. Handler: m.server,
  134. PickHost: m.pickTunnelHost,
  135. InsecureTLS: m.config.InsecureTLS,
  136. OnStateChange: m.onTunnelState,
  137. }
  138. m.mu.Lock()
  139. m.refreshLocalRecordLocked()
  140. m.mu.Unlock()
  141. if m.cluster != nil {
  142. logger.PrintAndLog("Cluster", "Resuming membership of cluster "+m.cluster.Name+" ("+m.cluster.ID+")", nil)
  143. m.startLoops()
  144. }
  145. return m, nil
  146. }
  147. // Close stops heartbeats and tunnels and closes the database.
  148. func (m *Manager) Close() {
  149. m.stopLoops()
  150. m.hub.Close()
  151. m.store.close()
  152. }
  153. // ACNHandler is the HTTP handler to mount at acn.BasePath.
  154. func (m *Manager) ACNHandler() http.Handler {
  155. return m.server
  156. }
  157. // Transport exposes the node-to-node transport to higher layers.
  158. func (m *Manager) Transport() *acn.Transport {
  159. return m.transport
  160. }
  161. // Server exposes the ACN server so higher layers can register endpoints.
  162. func (m *Manager) Server() *acn.Server {
  163. return m.server
  164. }
  165. // NodeID returns the ID of this node.
  166. func (m *Manager) NodeID() string {
  167. return m.opt.NodeID
  168. }
  169. // Sign signs an arbitrary message with this node's key so higher layers can
  170. // issue verifiable statements (e.g. user assertions).
  171. func (m *Manager) Sign(message []byte) []byte {
  172. return ed25519.Sign(m.key.Private, message)
  173. }
  174. // DB exposes the cluster database so sibling cluster services can keep their
  175. // cluster-scoped state in it (use the TableIdentity table, wiped on leave).
  176. func (m *Manager) DB() *database.Database {
  177. return m.store.db
  178. }
  179. // NodeName returns the display name of a member, or the ID when unknown.
  180. func (m *Manager) NodeName(nodeID string) string {
  181. m.mu.RLock()
  182. defer m.mu.RUnlock()
  183. if rec, ok := m.nodes[nodeID]; ok && rec.Name != "" {
  184. return rec.Name
  185. }
  186. return nodeID
  187. }
  188. // IdentityOrigin returns the node that verifies logins for the cluster,
  189. // empty when unset or when not in a cluster.
  190. func (m *Manager) IdentityOrigin() string {
  191. m.mu.RLock()
  192. defer m.mu.RUnlock()
  193. if m.cluster == nil {
  194. return ""
  195. }
  196. return m.cluster.IdentityOrigin
  197. }
  198. // SetIdentityOrigin makes nodeID the login authority of the cluster (empty
  199. // disables forward authentication) and replicates the setting.
  200. func (m *Manager) SetIdentityOrigin(nodeID string) error {
  201. m.mu.Lock()
  202. if m.cluster == nil {
  203. m.mu.Unlock()
  204. return ErrNotInCluster
  205. }
  206. if nodeID != "" {
  207. rec, ok := m.nodes[nodeID]
  208. if !ok || rec.Removed {
  209. m.mu.Unlock()
  210. return ErrNodeNotFound
  211. }
  212. }
  213. m.cluster.IdentityOrigin = nodeID
  214. m.cluster.SettingsVersion = nextVersion(m.cluster.SettingsVersion)
  215. if err := m.store.saveCluster(m.cluster); err != nil {
  216. m.mu.Unlock()
  217. return err
  218. }
  219. peers := m.peerIDsLocked()
  220. cb := m.OnClusterChange
  221. m.mu.Unlock()
  222. if cb != nil {
  223. cb()
  224. }
  225. go m.broadcast(peers, acn.BasePath+"/members/sync", m.syncPayload(), 15*time.Second)
  226. return nil
  227. }
  228. // mergeClusterLocked adopts newer replicated cluster settings.
  229. func (m *Manager) mergeClusterLocked(in *ClusterInfo) bool {
  230. if in == nil || m.cluster == nil || in.ID != m.cluster.ID {
  231. return false
  232. }
  233. if in.SettingsVersion <= m.cluster.SettingsVersion {
  234. return false
  235. }
  236. m.cluster.IdentityOrigin = in.IdentityOrigin
  237. m.cluster.SettingsVersion = in.SettingsVersion
  238. if in.Name != "" {
  239. m.cluster.Name = in.Name
  240. }
  241. m.store.saveCluster(m.cluster)
  242. return true
  243. }
  244. // clusterCopyLocked returns a copy of the cluster info for gossip payloads.
  245. func (m *Manager) clusterCopyLocked() *ClusterInfo {
  246. if m.cluster == nil {
  247. return nil
  248. }
  249. c := *m.cluster
  250. return &c
  251. }
  252. // syncPayload builds the full gossip payload (settings plus every record).
  253. func (m *Manager) syncPayload() SyncRequest {
  254. m.mu.RLock()
  255. defer m.mu.RUnlock()
  256. return SyncRequest{Cluster: m.clusterCopyLocked(), Nodes: m.allRecordsLocked()}
  257. }
  258. // InCluster reports whether this node is currently a cluster member.
  259. func (m *Manager) InCluster() bool {
  260. m.mu.RLock()
  261. defer m.mu.RUnlock()
  262. return m.cluster != nil
  263. }
  264. // Cluster returns a copy of the cluster info, nil when standalone.
  265. func (m *Manager) Cluster() *ClusterInfo {
  266. m.mu.RLock()
  267. defer m.mu.RUnlock()
  268. if m.cluster == nil {
  269. return nil
  270. }
  271. c := *m.cluster
  272. return &c
  273. }
  274. func (m *Manager) clusterID() string {
  275. m.mu.RLock()
  276. defer m.mu.RUnlock()
  277. if m.cluster == nil {
  278. return ""
  279. }
  280. return m.cluster.ID
  281. }
  282. // ResolvePeer implements acn.PeerResolver.
  283. func (m *Manager) ResolvePeer(nodeID string) (*acn.Peer, bool) {
  284. m.mu.RLock()
  285. defer m.mu.RUnlock()
  286. rec, ok := m.nodes[nodeID]
  287. if !ok || rec.Removed {
  288. return nil, false
  289. }
  290. pub, err := acn.DecodePublicKey(rec.PublicKey)
  291. if err != nil {
  292. return nil, false
  293. }
  294. return &acn.Peer{
  295. ID: rec.ID,
  296. Name: rec.Name,
  297. PublicKey: pub,
  298. AdvertiseURL: rec.AdvertiseURL,
  299. TunnelVia: rec.TunnelVia,
  300. }, true
  301. }
  302. /*
  303. Local record
  304. */
  305. // refreshLocalRecordLocked makes sure the record describing this node exists
  306. // and reflects the current config, key and capabilities.
  307. func (m *Manager) refreshLocalRecordLocked() {
  308. now := time.Now().Unix()
  309. rec, ok := m.nodes[m.opt.NodeID]
  310. if !ok {
  311. rec = &NodeRecord{ID: m.opt.NodeID, Joined: now}
  312. m.nodes[m.opt.NodeID] = rec
  313. }
  314. changed := !ok || rec.Removed
  315. url := strings.TrimRight(strings.TrimSpace(m.config.AdvertiseURL), "/")
  316. if rec.Name != m.config.Name || rec.PublicKey != m.key.PublicKeyString() || rec.AdvertiseURL != url || rec.Version != m.opt.Version {
  317. changed = true
  318. }
  319. if rec.Capabilities.DetectedAt != m.caps.DetectedAt {
  320. changed = true
  321. }
  322. if url != "" && rec.TunnelVia != "" {
  323. //A reachable node never needs a tunnel host
  324. rec.TunnelVia = ""
  325. changed = true
  326. }
  327. if changed {
  328. rec.Name = m.config.Name
  329. rec.PublicKey = m.key.PublicKeyString()
  330. rec.AdvertiseURL = url
  331. rec.Version = m.opt.Version
  332. rec.Capabilities = m.caps
  333. rec.Removed = false
  334. rec.Updated = nextVersion(rec.Updated)
  335. m.store.saveNode(rec)
  336. }
  337. }
  338. // localSnapshotLocked returns the local record with fresh transient fields.
  339. func (m *Manager) localSnapshotLocked() NodeRecord {
  340. rec := m.nodes[m.opt.NodeID].Clone()
  341. h := m.opt.Health()
  342. h.Uptime = int64(time.Since(m.started).Seconds())
  343. h.Timestamp = time.Now().Unix()
  344. rec.Health = h
  345. rec.LastSeen = h.Timestamp
  346. return *rec
  347. }
  348. // allRecordsLocked lists every record including tombstones, self first.
  349. func (m *Manager) allRecordsLocked() []NodeRecord {
  350. out := []NodeRecord{m.localSnapshotLocked()}
  351. ids := make([]string, 0, len(m.nodes))
  352. for id := range m.nodes {
  353. if id != m.opt.NodeID {
  354. ids = append(ids, id)
  355. }
  356. }
  357. sort.Strings(ids)
  358. for _, id := range ids {
  359. out = append(out, *m.nodes[id].Clone())
  360. }
  361. return out
  362. }
  363. func (m *Manager) peerIDsLocked() []string {
  364. ids := []string{}
  365. for id, rec := range m.nodes {
  366. if id != m.opt.NodeID && !rec.Removed {
  367. ids = append(ids, id)
  368. }
  369. }
  370. sort.Strings(ids)
  371. return ids
  372. }
  373. /*
  374. Merge
  375. */
  376. // mergeRecordLocked folds one gossiped record into the local view. It reports
  377. // whether persistent fields changed and whether this node has been evicted.
  378. func (m *Manager) mergeRecordLocked(in NodeRecord) (changed bool, evicted bool) {
  379. if in.ID == "" {
  380. return false, false
  381. }
  382. if in.ID == m.opt.NodeID {
  383. local := m.nodes[m.opt.NodeID]
  384. if in.Updated > local.Updated {
  385. if in.Removed {
  386. return false, true
  387. }
  388. //Only admin state may be set on us by other members
  389. if in.AdminState != local.AdminState {
  390. local.AdminState = in.AdminState
  391. local.Updated = in.Updated
  392. return true, false
  393. }
  394. }
  395. return false, false
  396. }
  397. existing, ok := m.nodes[in.ID]
  398. if !ok {
  399. rec := in
  400. m.nodes[in.ID] = &rec
  401. return true, false
  402. }
  403. if in.Updated > existing.Updated {
  404. existing.Name = in.Name
  405. existing.PublicKey = in.PublicKey
  406. existing.AdvertiseURL = in.AdvertiseURL
  407. existing.TunnelVia = in.TunnelVia
  408. existing.Version = in.Version
  409. existing.Capabilities = in.Capabilities
  410. existing.AdminState = in.AdminState
  411. existing.Removed = in.Removed
  412. existing.Joined = in.Joined
  413. existing.Updated = in.Updated
  414. changed = true
  415. }
  416. if in.Health.Timestamp > existing.Health.Timestamp {
  417. existing.Health = in.Health
  418. }
  419. if in.LastSeen > existing.LastSeen {
  420. existing.LastSeen = in.LastSeen
  421. }
  422. return changed, false
  423. }
  424. // mergeNodes applies a batch of records, persists changes and handles eviction.
  425. func (m *Manager) mergeNodes(records []NodeRecord) {
  426. m.mergeGossip(nil, records)
  427. }
  428. // mergeGossip applies replicated cluster settings and a batch of records.
  429. func (m *Manager) mergeGossip(cluster *ClusterInfo, records []NodeRecord) {
  430. m.mu.Lock()
  431. if m.cluster == nil {
  432. m.mu.Unlock()
  433. return
  434. }
  435. settingsChanged := m.mergeClusterLocked(cluster)
  436. evicted := false
  437. for _, in := range records {
  438. changed, ev := m.mergeRecordLocked(in)
  439. if ev {
  440. evicted = true
  441. }
  442. if changed {
  443. m.store.saveNode(m.nodes[in.ID])
  444. }
  445. }
  446. m.gcTombstonesLocked()
  447. cb := m.OnClusterChange
  448. m.mu.Unlock()
  449. if settingsChanged && cb != nil {
  450. cb()
  451. }
  452. if evicted {
  453. logger.PrintAndLog("Cluster", "This node has been removed from the cluster by another member", nil)
  454. m.wipeLocalState()
  455. }
  456. }
  457. func (m *Manager) gcTombstonesLocked() {
  458. cutoff := time.Now().Add(-TombstoneTTL).UnixMilli()
  459. for id, rec := range m.nodes {
  460. if rec.Removed && rec.Updated < cutoff && id != m.opt.NodeID {
  461. delete(m.nodes, id)
  462. m.store.deleteNode(id)
  463. }
  464. }
  465. }
  466. /*
  467. Cluster life cycle
  468. */
  469. // CreateCluster turns this standalone node into the first member of a new cluster.
  470. func (m *Manager) CreateCluster(name string) (*ClusterInfo, error) {
  471. name = strings.TrimSpace(name)
  472. if name == "" {
  473. return nil, errors.New("cluster name cannot be empty")
  474. }
  475. m.mu.Lock()
  476. if m.cluster != nil {
  477. m.mu.Unlock()
  478. return nil, ErrAlreadyInCluster
  479. }
  480. info := &ClusterInfo{ID: uuid.NewV4().String(), Name: name, Created: time.Now().Unix()}
  481. if err := m.store.saveCluster(info); err != nil {
  482. m.mu.Unlock()
  483. return nil, err
  484. }
  485. m.cluster = info
  486. m.signer.ClusterID = info.ID
  487. m.nodes = map[string]*NodeRecord{}
  488. m.refreshLocalRecordLocked()
  489. local := m.nodes[m.opt.NodeID]
  490. local.Joined = info.Created
  491. local.Updated = nextVersion(local.Updated)
  492. m.store.saveNode(local)
  493. m.mu.Unlock()
  494. logger.PrintAndLog("Cluster", "Created cluster "+name+" ("+info.ID+")", nil)
  495. m.startLoops()
  496. m.fireMembershipChange(true)
  497. c := *info
  498. return &c, nil
  499. }
  500. // JoinCluster contacts the issuing node named in the token and joins its cluster.
  501. func (m *Manager) JoinCluster(tokenString string) (*ClusterInfo, error) {
  502. payload, err := DecodeJoinToken(tokenString)
  503. if err != nil {
  504. return nil, err
  505. }
  506. m.mu.Lock()
  507. if m.cluster != nil {
  508. m.mu.Unlock()
  509. return nil, ErrAlreadyInCluster
  510. }
  511. m.refreshLocalRecordLocked()
  512. now := time.Now().Unix()
  513. local := m.nodes[m.opt.NodeID].Clone()
  514. local.Joined = now
  515. local.Updated = nextVersion(local.Updated)
  516. local.AdminState = AdminStateNormal
  517. req := JoinRequest{ClusterID: payload.ClusterID, TokenID: payload.TokenID, Secret: payload.Secret, Node: *local}
  518. m.mu.Unlock()
  519. js, _ := json.Marshal(req)
  520. ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
  521. defer cancel()
  522. resp, err := m.transport.DoURL(ctx, payload.URL, http.MethodPost, acn.BasePath+"/join", js, false)
  523. if err != nil {
  524. return nil, errors.New("unable to reach " + payload.URL + ": " + err.Error())
  525. }
  526. if err := resp.Error(); err != nil {
  527. return nil, err
  528. }
  529. var jr JoinResponse
  530. if err := json.Unmarshal(resp.Body, &jr); err != nil {
  531. return nil, errors.New("invalid join response from " + payload.URL)
  532. }
  533. if jr.Cluster.ID == "" || jr.Cluster.ID != payload.ClusterID {
  534. return nil, errors.New("join response does not match the token's cluster")
  535. }
  536. m.mu.Lock()
  537. if m.cluster != nil {
  538. m.mu.Unlock()
  539. return nil, ErrAlreadyInCluster
  540. }
  541. info := jr.Cluster
  542. if err := m.store.saveCluster(&info); err != nil {
  543. m.mu.Unlock()
  544. return nil, err
  545. }
  546. m.cluster = &info
  547. m.signer.ClusterID = info.ID
  548. m.nodes = map[string]*NodeRecord{}
  549. self := *local
  550. if self.AdvertiseURL == "" {
  551. self.TunnelVia = jr.Via
  552. }
  553. m.nodes[self.ID] = &self
  554. m.store.saveNode(&self)
  555. for _, rec := range jr.Nodes {
  556. if changed, _ := m.mergeRecordLocked(rec); changed {
  557. m.store.saveNode(m.nodes[rec.ID])
  558. }
  559. }
  560. m.mu.Unlock()
  561. logger.PrintAndLog("Cluster", "Joined cluster "+info.Name+" ("+info.ID+") through "+payload.URL, nil)
  562. m.startLoops()
  563. m.fireMembershipChange(true)
  564. return &info, nil
  565. }
  566. // LeaveCluster tells the other members goodbye and returns to standalone mode.
  567. func (m *Manager) LeaveCluster() error {
  568. m.mu.RLock()
  569. if m.cluster == nil {
  570. m.mu.RUnlock()
  571. return ErrNotInCluster
  572. }
  573. peers := m.peerIDsLocked()
  574. m.mu.RUnlock()
  575. m.stopLoops()
  576. m.broadcast(peers, acn.BasePath+"/leave", NodeIDRequest{NodeID: m.opt.NodeID}, 10*time.Second)
  577. m.wipeLocalState()
  578. logger.PrintAndLog("Cluster", "Left the cluster", nil)
  579. return nil
  580. }
  581. // wipeLocalState drops all cluster records and returns to standalone mode.
  582. func (m *Manager) wipeLocalState() {
  583. m.stopLoops()
  584. m.hub.Close()
  585. m.mu.Lock()
  586. m.store.wipeCluster()
  587. m.cluster = nil
  588. m.signer.ClusterID = ""
  589. m.nodes = map[string]*NodeRecord{}
  590. m.tokens = map[string]*JoinToken{}
  591. m.reachable = map[string]bool{}
  592. m.latency = map[string]float64{}
  593. m.latencyMx.reset()
  594. m.refreshLocalRecordLocked()
  595. m.mu.Unlock()
  596. m.fireMembershipChange(false)
  597. }
  598. // RemoveNode evicts another member.
  599. func (m *Manager) RemoveNode(nodeID string) error {
  600. if nodeID == m.opt.NodeID {
  601. return errors.New("use leave to remove this node")
  602. }
  603. m.mu.Lock()
  604. if m.cluster == nil {
  605. m.mu.Unlock()
  606. return ErrNotInCluster
  607. }
  608. rec, ok := m.nodes[nodeID]
  609. if !ok || rec.Removed {
  610. m.mu.Unlock()
  611. return ErrNodeNotFound
  612. }
  613. rec.Removed = true
  614. rec.Updated = nextVersion(rec.Updated)
  615. m.store.saveNode(rec)
  616. peers := m.peerIDsLocked()
  617. m.mu.Unlock()
  618. go func() {
  619. ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
  620. defer cancel()
  621. //Tell the evicted node itself first; ResolvePeer no longer knows it, so resolve manually
  622. m.mu.RLock()
  623. target := m.nodes[nodeID]
  624. m.mu.RUnlock()
  625. if target != nil {
  626. m.notifyEvicted(ctx, target)
  627. }
  628. m.broadcast(peers, acn.BasePath+"/members/sync", m.syncPayload(), 15*time.Second)
  629. }()
  630. logger.PrintAndLog("Cluster", "Removed node "+nodeID+" from the cluster", nil)
  631. return nil
  632. }
  633. // notifyEvicted delivers the evict notice to a node that is no longer resolvable.
  634. func (m *Manager) notifyEvicted(ctx context.Context, target *NodeRecord) {
  635. body, _ := json.Marshal(NodeIDRequest{NodeID: target.ID})
  636. if target.AdvertiseURL != "" {
  637. m.transport.DoURL(ctx, target.AdvertiseURL, http.MethodPost, acn.BasePath+"/evict", body, true)
  638. return
  639. }
  640. if m.hub.Connected(target.ID) {
  641. req, _ := http.NewRequest(http.MethodPost, acn.BasePath+"/evict", nil)
  642. req.Header.Set("Content-Type", "application/json")
  643. m.signer.Sign(req, acn.BasePath+"/evict", body)
  644. m.hub.Do(ctx, target.ID, http.MethodPost, acn.BasePath+"/evict", req.Header, body)
  645. }
  646. }
  647. // SetNodeAdminState puts a node into maintenance / draining / normal mode.
  648. func (m *Manager) SetNodeAdminState(nodeID string, state string) error {
  649. state = strings.ToLower(strings.TrimSpace(state))
  650. if state == "normal" {
  651. state = AdminStateNormal
  652. }
  653. if state != AdminStateNormal && state != AdminStateMaintenance && state != AdminStateDraining {
  654. return errors.New("unsupported node state")
  655. }
  656. m.mu.Lock()
  657. if m.cluster == nil {
  658. m.mu.Unlock()
  659. return ErrNotInCluster
  660. }
  661. rec, ok := m.nodes[nodeID]
  662. if !ok || rec.Removed {
  663. m.mu.Unlock()
  664. return ErrNodeNotFound
  665. }
  666. rec.AdminState = state
  667. rec.Updated = nextVersion(rec.Updated)
  668. m.store.saveNode(rec)
  669. peers := m.peerIDsLocked()
  670. m.mu.Unlock()
  671. go m.broadcast(peers, acn.BasePath+"/members/sync", m.syncPayload(), 15*time.Second)
  672. return nil
  673. }
  674. /*
  675. Configuration
  676. */
  677. // Config returns the local node configuration.
  678. func (m *Manager) Config() LocalConfig {
  679. m.mu.RLock()
  680. defer m.mu.RUnlock()
  681. return m.config
  682. }
  683. // UpdateConfig changes the local node configuration and republishes the record.
  684. func (m *Manager) UpdateConfig(cfg LocalConfig) error {
  685. cfg.Name = strings.TrimSpace(cfg.Name)
  686. cfg.AdvertiseURL = strings.TrimRight(strings.TrimSpace(cfg.AdvertiseURL), "/")
  687. cfg.TunnelVia = strings.TrimSpace(cfg.TunnelVia)
  688. if cfg.Name == "" {
  689. return errors.New("node name cannot be empty")
  690. }
  691. if cfg.AdvertiseURL != "" && !strings.HasPrefix(cfg.AdvertiseURL, "http://") && !strings.HasPrefix(cfg.AdvertiseURL, "https://") {
  692. return errors.New("advertise URL must start with http:// or https://")
  693. }
  694. if cfg.TunnelVia == m.opt.NodeID {
  695. cfg.TunnelVia = ""
  696. }
  697. m.mu.Lock()
  698. old := m.config
  699. m.config = cfg
  700. if err := m.store.saveConfig(cfg); err != nil {
  701. m.config = old
  702. m.mu.Unlock()
  703. return err
  704. }
  705. m.transport.SetInsecureTLS(cfg.InsecureTLS)
  706. m.tunnel.InsecureTLS = cfg.InsecureTLS
  707. m.refreshLocalRecordLocked()
  708. inCluster := m.cluster != nil
  709. peers := m.peerIDsLocked()
  710. m.mu.Unlock()
  711. if inCluster {
  712. switch {
  713. case old.AdvertiseURL == "" && cfg.AdvertiseURL != "":
  714. m.tunnel.Stop()
  715. case old.AdvertiseURL != "" && cfg.AdvertiseURL == "":
  716. m.tunnel.Start()
  717. case cfg.AdvertiseURL == "" && old.TunnelVia != cfg.TunnelVia:
  718. m.tunnel.Reconnect()
  719. }
  720. go m.broadcast(peers, acn.BasePath+"/members/sync", m.syncPayload(), 15*time.Second)
  721. }
  722. return nil
  723. }
  724. // RefreshCapabilities re-detects the local manifest and republishes it.
  725. func (m *Manager) RefreshCapabilities() capability.Manifest {
  726. caps := m.opt.Capabilities()
  727. m.mu.Lock()
  728. m.caps = caps
  729. m.refreshLocalRecordLocked()
  730. m.mu.Unlock()
  731. return caps
  732. }
  733. /*
  734. Join tokens
  735. */
  736. // NewJoinToken issues a token other nodes can use to join through this node.
  737. func (m *Manager) NewJoinToken(ttl time.Duration) (string, *JoinToken, error) {
  738. if ttl <= 0 {
  739. ttl = 24 * time.Hour
  740. }
  741. m.mu.Lock()
  742. defer m.mu.Unlock()
  743. if m.cluster == nil {
  744. return "", nil, ErrNotInCluster
  745. }
  746. token, encoded, err := NewJoinToken(*m.cluster, m.config.AdvertiseURL, ttl)
  747. if err != nil {
  748. return "", nil, err
  749. }
  750. if err := m.store.saveJoinToken(token); err != nil {
  751. return "", nil, err
  752. }
  753. m.tokens[token.ID] = token
  754. return encoded, token, nil
  755. }
  756. // ListJoinTokens returns the unexpired tokens, purging expired ones.
  757. func (m *Manager) ListJoinTokens() []JoinToken {
  758. m.mu.Lock()
  759. defer m.mu.Unlock()
  760. now := time.Now().Unix()
  761. out := []JoinToken{}
  762. for id, t := range m.tokens {
  763. if t.Expires < now {
  764. delete(m.tokens, id)
  765. m.store.deleteJoinToken(id)
  766. continue
  767. }
  768. out = append(out, *t)
  769. }
  770. sort.Slice(out, func(i, j int) bool { return out[i].Created > out[j].Created })
  771. return out
  772. }
  773. // RevokeJoinToken deletes a token before it expires.
  774. func (m *Manager) RevokeJoinToken(id string) error {
  775. m.mu.Lock()
  776. defer m.mu.Unlock()
  777. if _, ok := m.tokens[id]; !ok {
  778. return errors.New("token not found")
  779. }
  780. delete(m.tokens, id)
  781. return m.store.deleteJoinToken(id)
  782. }
  783. /*
  784. Views
  785. */
  786. // NodeViews lists the live members with their computed state.
  787. func (m *Manager) NodeViews() []NodeView {
  788. m.mu.RLock()
  789. defer m.mu.RUnlock()
  790. now := time.Now()
  791. out := []NodeView{}
  792. for _, rec := range m.allRecordsLocked() {
  793. if rec.Removed {
  794. continue
  795. }
  796. local := rec.ID == m.opt.NodeID
  797. lat := -1.0
  798. if local {
  799. lat = 0
  800. } else if v, ok := m.latency[rec.ID]; ok {
  801. lat = v
  802. }
  803. out = append(out, NodeView{
  804. NodeRecord: rec,
  805. State: rec.ComputeState(now, local),
  806. Local: local,
  807. Tunnel: m.hub.Connected(rec.ID),
  808. LatencyMs: lat,
  809. })
  810. }
  811. return out
  812. }
  813. // snapshotRecords returns every record for gossip.
  814. func (m *Manager) snapshotRecords() []NodeRecord {
  815. m.mu.RLock()
  816. defer m.mu.RUnlock()
  817. return m.allRecordsLocked()
  818. }
  819. // LocalView summarises this node for the settings UI.
  820. type LocalView struct {
  821. NodeID string `json:"nodeId"`
  822. PublicKey string `json:"publicKey"`
  823. Config LocalConfig `json:"config"`
  824. Version string `json:"version"`
  825. Capabilities capability.Manifest `json:"capabilities"`
  826. Health Health `json:"health"`
  827. Reachable bool `json:"reachable"`
  828. TunnelConnected bool `json:"tunnelConnected"`
  829. TunnelHost string `json:"tunnelHost"`
  830. }
  831. // Status is the full picture handed to the admin UI.
  832. type Status struct {
  833. InCluster bool `json:"inCluster"`
  834. Cluster *ClusterInfo `json:"cluster"`
  835. Local LocalView `json:"local"`
  836. Nodes []NodeView `json:"nodes"`
  837. Tokens []JoinToken `json:"tokens"`
  838. Tunnels []string `json:"tunnels"`
  839. Time int64 `json:"time"`
  840. }
  841. // Status builds the admin view.
  842. func (m *Manager) Status() Status {
  843. connected, host := m.tunnel.Status()
  844. m.mu.RLock()
  845. local := m.localSnapshotLocked()
  846. cfg := m.config
  847. var cluster *ClusterInfo
  848. if m.cluster != nil {
  849. c := *m.cluster
  850. cluster = &c
  851. }
  852. m.mu.RUnlock()
  853. return Status{
  854. InCluster: cluster != nil,
  855. Cluster: cluster,
  856. Local: LocalView{
  857. NodeID: m.opt.NodeID,
  858. PublicKey: m.key.PublicKeyString(),
  859. Config: cfg,
  860. Version: m.opt.Version,
  861. Capabilities: local.Capabilities,
  862. Health: local.Health,
  863. Reachable: cfg.AdvertiseURL != "",
  864. TunnelConnected: connected,
  865. TunnelHost: host.NodeID,
  866. },
  867. Nodes: m.NodeViews(),
  868. Tokens: m.ListJoinTokens(),
  869. Tunnels: m.hub.ConnectedNodes(),
  870. Time: time.Now().Unix(),
  871. }
  872. }
  873. // ProbeResult is the outcome of an on-demand reachability check.
  874. type ProbeResult struct {
  875. NodeID string `json:"nodeId"`
  876. OK bool `json:"ok"`
  877. LatencyMs int64 `json:"latencyMs"`
  878. Route string `json:"route"`
  879. Error string `json:"error,omitempty"`
  880. }
  881. // ProbeNode pings a member through whatever route the transport picks.
  882. func (m *Manager) ProbeNode(nodeID string) ProbeResult {
  883. res := ProbeResult{NodeID: nodeID}
  884. peer, ok := m.ResolvePeer(nodeID)
  885. if !ok {
  886. res.Error = ErrNodeNotFound.Error()
  887. return res
  888. }
  889. switch {
  890. case m.hub.Connected(nodeID):
  891. res.Route = "tunnel"
  892. case peer.AdvertiseURL != "":
  893. res.Route = "direct"
  894. case peer.TunnelVia != "":
  895. res.Route = "relay via " + peer.TunnelVia
  896. default:
  897. res.Route = "none"
  898. }
  899. ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
  900. defer cancel()
  901. start := time.Now()
  902. err := m.transport.DoJSON(ctx, nodeID, http.MethodPost, acn.BasePath+"/ping", nil, nil)
  903. res.LatencyMs = time.Since(start).Milliseconds()
  904. if err != nil {
  905. res.Error = err.Error()
  906. return res
  907. }
  908. res.OK = true
  909. m.markSeen(nodeID)
  910. return res
  911. }
  912. /*
  913. Gossip loops
  914. */
  915. func (m *Manager) startLoops() {
  916. m.loopMu.Lock()
  917. if m.loopStop != nil {
  918. m.loopMu.Unlock()
  919. return
  920. }
  921. m.loopStop = make(chan struct{})
  922. stop := m.loopStop
  923. m.loopWG.Add(1)
  924. m.loopMu.Unlock()
  925. go func() {
  926. defer m.loopWG.Done()
  927. m.heartbeatAll()
  928. ticker := time.NewTicker(HeartbeatInterval)
  929. defer ticker.Stop()
  930. for {
  931. select {
  932. case <-stop:
  933. return
  934. case <-ticker.C:
  935. m.heartbeatAll()
  936. }
  937. }
  938. }()
  939. if m.Config().AdvertiseURL == "" {
  940. m.tunnel.Start()
  941. }
  942. }
  943. func (m *Manager) stopLoops() {
  944. m.loopMu.Lock()
  945. if m.loopStop == nil {
  946. m.loopMu.Unlock()
  947. return
  948. }
  949. close(m.loopStop)
  950. m.loopStop = nil
  951. m.loopMu.Unlock()
  952. m.loopWG.Wait()
  953. m.tunnel.Stop()
  954. }
  955. // heartbeatAll reports to every peer in parallel and merges their views.
  956. func (m *Manager) heartbeatAll() {
  957. m.mu.RLock()
  958. if m.cluster == nil {
  959. m.mu.RUnlock()
  960. return
  961. }
  962. local := m.localSnapshotLocked()
  963. cluster := m.clusterCopyLocked()
  964. peers := m.peerIDsLocked()
  965. m.mu.RUnlock()
  966. var wg sync.WaitGroup
  967. for _, id := range peers {
  968. wg.Add(1)
  969. go func(peerID string) {
  970. defer wg.Done()
  971. ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
  972. defer cancel()
  973. var resp HeartbeatResponse
  974. start := time.Now()
  975. err := m.transport.DoJSON(ctx, peerID, http.MethodPost, acn.BasePath+"/heartbeat", HeartbeatRequest{Node: local, Cluster: cluster}, &resp)
  976. m.noteReachability(peerID, err)
  977. if err != nil {
  978. return
  979. }
  980. m.noteLatency(peerID, float64(time.Since(start).Microseconds())/1000)
  981. m.markSeen(peerID)
  982. m.mergeGossip(resp.Cluster, resp.Nodes)
  983. }(id)
  984. }
  985. wg.Wait()
  986. }
  987. func (m *Manager) markSeen(nodeID string) {
  988. m.mu.Lock()
  989. defer m.mu.Unlock()
  990. if rec, ok := m.nodes[nodeID]; ok {
  991. rec.LastSeen = time.Now().Unix()
  992. }
  993. }
  994. // noteLatency folds a round trip measurement into the peer's moving average.
  995. func (m *Manager) noteLatency(peerID string, ms float64) {
  996. if ms < 0 {
  997. return
  998. }
  999. m.mu.Lock()
  1000. defer m.mu.Unlock()
  1001. if prev, ok := m.latency[peerID]; ok {
  1002. m.latency[peerID] = prev*(1-latencyAlpha) + ms*latencyAlpha
  1003. } else {
  1004. m.latency[peerID] = ms
  1005. }
  1006. }
  1007. // Latency returns the moving average round trip to a peer in milliseconds.
  1008. // known is false when this node has never measured it (0 for the local node).
  1009. func (m *Manager) Latency(nodeID string) (ms float64, known bool) {
  1010. if nodeID == m.opt.NodeID {
  1011. return 0, true
  1012. }
  1013. m.mu.RLock()
  1014. defer m.mu.RUnlock()
  1015. v, ok := m.latency[nodeID]
  1016. return v, ok
  1017. }
  1018. // Latencies returns a copy of every measured round trip.
  1019. func (m *Manager) Latencies() map[string]float64 {
  1020. m.mu.RLock()
  1021. defer m.mu.RUnlock()
  1022. out := map[string]float64{}
  1023. for k, v := range m.latency {
  1024. out[k] = v
  1025. }
  1026. return out
  1027. }
  1028. // noteReachability logs only when a peer flips between reachable and not.
  1029. func (m *Manager) noteReachability(peerID string, err error) {
  1030. m.mu.Lock()
  1031. prev, known := m.reachable[peerID]
  1032. now := err == nil
  1033. m.reachable[peerID] = now
  1034. name := peerID
  1035. if rec, ok := m.nodes[peerID]; ok && rec.Name != "" {
  1036. name = rec.Name
  1037. }
  1038. m.mu.Unlock()
  1039. if !known || prev != now {
  1040. if now {
  1041. logger.PrintAndLog("Cluster", "Node "+name+" is reachable", nil)
  1042. } else {
  1043. logger.PrintAndLog("Cluster", "Node "+name+" is unreachable: "+err.Error(), nil)
  1044. }
  1045. }
  1046. }
  1047. // broadcast posts payload to the given peers in parallel and waits for all.
  1048. func (m *Manager) broadcast(peers []string, path string, payload interface{}, timeout time.Duration) {
  1049. var wg sync.WaitGroup
  1050. for _, id := range peers {
  1051. wg.Add(1)
  1052. go func(peerID string) {
  1053. defer wg.Done()
  1054. ctx, cancel := context.WithTimeout(context.Background(), timeout)
  1055. defer cancel()
  1056. m.transport.DoJSON(ctx, peerID, http.MethodPost, path, payload, nil)
  1057. }(id)
  1058. }
  1059. wg.Wait()
  1060. }
  1061. /*
  1062. Tunnel callbacks
  1063. */
  1064. // pickTunnelHost chooses the reachable peer a NAT-only node should attach to.
  1065. func (m *Manager) pickTunnelHost() (acn.TunnelHost, bool) {
  1066. m.mu.RLock()
  1067. defer m.mu.RUnlock()
  1068. if m.cluster == nil {
  1069. return acn.TunnelHost{}, false
  1070. }
  1071. if pref := m.config.TunnelVia; pref != "" {
  1072. if rec, ok := m.nodes[pref]; ok && !rec.Removed && rec.AdvertiseURL != "" {
  1073. return acn.TunnelHost{NodeID: rec.ID, URL: rec.AdvertiseURL}, true
  1074. }
  1075. }
  1076. now := time.Now()
  1077. var best *NodeRecord
  1078. for _, id := range m.peerIDsLocked() {
  1079. rec := m.nodes[id]
  1080. if rec.AdvertiseURL == "" {
  1081. continue
  1082. }
  1083. if best == nil {
  1084. best = rec
  1085. continue
  1086. }
  1087. bs, rs := best.ComputeState(now, false), rec.ComputeState(now, false)
  1088. if bs != StateOnline && rs == StateOnline {
  1089. best = rec
  1090. } else if bs == rs && rec.LastSeen > best.LastSeen {
  1091. best = rec
  1092. }
  1093. }
  1094. if best == nil {
  1095. return acn.TunnelHost{}, false
  1096. }
  1097. return acn.TunnelHost{NodeID: best.ID, URL: best.AdvertiseURL}, true
  1098. }
  1099. // onTunnelState records which host now terminates our tunnel and tells it.
  1100. func (m *Manager) onTunnelState(connected bool, host acn.TunnelHost) {
  1101. if !connected {
  1102. return
  1103. }
  1104. m.mu.Lock()
  1105. local, ok := m.nodes[m.opt.NodeID]
  1106. if ok && local.TunnelVia != host.NodeID {
  1107. local.TunnelVia = host.NodeID
  1108. local.Updated = nextVersion(local.Updated)
  1109. m.store.saveNode(local)
  1110. }
  1111. peers := m.peerIDsLocked()
  1112. m.mu.Unlock()
  1113. go m.broadcast(peers, acn.BasePath+"/members/sync", m.syncPayload(), 15*time.Second)
  1114. }
  1115. // onTunnelConnect runs on the host side when a NAT-only node attaches.
  1116. func (m *Manager) onTunnelConnect(nodeID string) {
  1117. m.mu.Lock()
  1118. defer m.mu.Unlock()
  1119. rec, ok := m.nodes[nodeID]
  1120. if !ok {
  1121. return
  1122. }
  1123. rec.LastSeen = time.Now().Unix()
  1124. if rec.TunnelVia != m.opt.NodeID {
  1125. rec.TunnelVia = m.opt.NodeID
  1126. rec.Updated = nextVersion(rec.Updated)
  1127. m.store.saveNode(rec)
  1128. }
  1129. }