tunnel.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640
  1. package acn
  2. /*
  3. ArozOS Cluster Node protocol (ACN) - reverse tunnel
  4. Nodes that sit behind NAT or otherwise have no public URL keep one
  5. persistent WebSocket connection open to a reachable peer (the "host").
  6. HTTP requests destined for the tunnelled node are multiplexed over that
  7. socket as binary frames, executed against the tunnelled node's local ACN
  8. handler, and the responses are sent back over the same socket.
  9. Frame layout (binary WebSocket message):
  10. [1 byte kind][4 byte big-endian header length][header JSON][body bytes]
  11. Requests are signed by the ORIGINAL sender, so the tunnelled node verifies
  12. the same signature the host (or a relaying node) already verified.
  13. Cloudflare terminates idle WebSockets after roughly 100 seconds, so both
  14. ends ping every TunnelPingInterval.
  15. */
  16. import (
  17. "bytes"
  18. "context"
  19. "encoding/binary"
  20. "encoding/json"
  21. "errors"
  22. "io"
  23. "net/http"
  24. "net/url"
  25. "strings"
  26. "sync"
  27. "time"
  28. "github.com/gorilla/websocket"
  29. "imuslab.com/arozos/mod/info/logger"
  30. )
  31. const (
  32. frameKindRequest byte = 1
  33. frameKindResponse byte = 2
  34. TunnelPath = BasePath + "/tunnel"
  35. TunnelPingInterval = 30 * time.Second
  36. TunnelReadTimeout = 90 * time.Second
  37. // TunnelMaxFrame bounds a single multiplexed request or response so a 4MB
  38. // transfer chunk plus headers always fits with room to spare.
  39. TunnelMaxFrame = 16 << 20
  40. )
  41. var (
  42. ErrTunnelNotConnected = errors.New("target node is not connected through a tunnel on this node")
  43. ErrTunnelClosed = errors.New("tunnel closed")
  44. ErrFrameTooLarge = errors.New("tunnel frame exceeds maximum size")
  45. )
  46. type frameHeader struct {
  47. ID string `json:"id"`
  48. Method string `json:"m,omitempty"`
  49. Target string `json:"t,omitempty"` //path plus query
  50. Status int `json:"s,omitempty"`
  51. Headers map[string][]string `json:"h,omitempty"`
  52. }
  53. func encodeFrame(kind byte, hdr frameHeader, body []byte) ([]byte, error) {
  54. hj, err := json.Marshal(hdr)
  55. if err != nil {
  56. return nil, err
  57. }
  58. if len(hj)+len(body)+5 > TunnelMaxFrame {
  59. return nil, ErrFrameTooLarge
  60. }
  61. buf := make([]byte, 5, 5+len(hj)+len(body))
  62. buf[0] = kind
  63. binary.BigEndian.PutUint32(buf[1:5], uint32(len(hj)))
  64. buf = append(buf, hj...)
  65. buf = append(buf, body...)
  66. return buf, nil
  67. }
  68. func decodeFrame(data []byte) (byte, frameHeader, []byte, error) {
  69. var hdr frameHeader
  70. if len(data) < 5 {
  71. return 0, hdr, nil, errors.New("tunnel frame too short")
  72. }
  73. kind := data[0]
  74. hlen := int(binary.BigEndian.Uint32(data[1:5]))
  75. if hlen < 0 || 5+hlen > len(data) {
  76. return 0, hdr, nil, errors.New("tunnel frame header length invalid")
  77. }
  78. if err := json.Unmarshal(data[5:5+hlen], &hdr); err != nil {
  79. return 0, hdr, nil, err
  80. }
  81. return kind, hdr, data[5+hlen:], nil
  82. }
  83. // Response is the buffered result of a node-to-node request.
  84. type Response struct {
  85. Status int
  86. Header http.Header
  87. Body []byte
  88. }
  89. // Error turns a non-2xx response into a descriptive error, nil otherwise.
  90. func (r *Response) Error() error {
  91. if r == nil {
  92. return errors.New("empty response")
  93. }
  94. if r.Status >= 200 && r.Status < 300 {
  95. return nil
  96. }
  97. msg := strings.TrimSpace(string(r.Body))
  98. if len(msg) > 200 {
  99. msg = msg[:200]
  100. }
  101. return errors.New("remote node returned " + http.StatusText(r.Status) + ": " + msg)
  102. }
  103. // tunnelConn is one live WebSocket shared by both ends of a tunnel.
  104. type tunnelConn struct {
  105. ws *websocket.Conn
  106. writeMu sync.Mutex
  107. pending map[string]chan *Response
  108. pmu sync.Mutex
  109. closed chan struct{}
  110. once sync.Once
  111. seq uint64
  112. }
  113. func newTunnelConn(ws *websocket.Conn) *tunnelConn {
  114. ws.SetReadLimit(TunnelMaxFrame)
  115. return &tunnelConn{
  116. ws: ws,
  117. pending: map[string]chan *Response{},
  118. closed: make(chan struct{}),
  119. }
  120. }
  121. func (c *tunnelConn) write(kind byte, hdr frameHeader, body []byte) error {
  122. frame, err := encodeFrame(kind, hdr, body)
  123. if err != nil {
  124. return err
  125. }
  126. c.writeMu.Lock()
  127. defer c.writeMu.Unlock()
  128. select {
  129. case <-c.closed:
  130. return ErrTunnelClosed
  131. default:
  132. }
  133. c.ws.SetWriteDeadline(time.Now().Add(60 * time.Second))
  134. return c.ws.WriteMessage(websocket.BinaryMessage, frame)
  135. }
  136. func (c *tunnelConn) ping() error {
  137. c.writeMu.Lock()
  138. defer c.writeMu.Unlock()
  139. return c.ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(10*time.Second))
  140. }
  141. func (c *tunnelConn) close() {
  142. c.once.Do(func() {
  143. close(c.closed)
  144. c.ws.Close()
  145. c.pmu.Lock()
  146. for id, ch := range c.pending {
  147. close(ch)
  148. delete(c.pending, id)
  149. }
  150. c.pmu.Unlock()
  151. })
  152. }
  153. func (c *tunnelConn) isClosed() bool {
  154. select {
  155. case <-c.closed:
  156. return true
  157. default:
  158. return false
  159. }
  160. }
  161. // roundTrip sends a request frame and waits for its response frame.
  162. func (c *tunnelConn) roundTrip(ctx context.Context, method string, target string, headers http.Header, body []byte) (*Response, error) {
  163. c.pmu.Lock()
  164. c.seq++
  165. id := newNonce()
  166. ch := make(chan *Response, 1)
  167. c.pending[id] = ch
  168. c.pmu.Unlock()
  169. cleanup := func() {
  170. c.pmu.Lock()
  171. delete(c.pending, id)
  172. c.pmu.Unlock()
  173. }
  174. err := c.write(frameKindRequest, frameHeader{
  175. ID: id,
  176. Method: method,
  177. Target: target,
  178. Headers: map[string][]string(headers),
  179. }, body)
  180. if err != nil {
  181. cleanup()
  182. return nil, err
  183. }
  184. select {
  185. case resp, ok := <-ch:
  186. cleanup()
  187. if !ok || resp == nil {
  188. return nil, ErrTunnelClosed
  189. }
  190. return resp, nil
  191. case <-ctx.Done():
  192. cleanup()
  193. return nil, ctx.Err()
  194. case <-c.closed:
  195. cleanup()
  196. return nil, ErrTunnelClosed
  197. }
  198. }
  199. // serveRequest executes an incoming request frame against handler and writes
  200. // the response frame back.
  201. func (c *tunnelConn) serveRequest(handler http.Handler, hdr frameHeader, body []byte) {
  202. u, err := url.ParseRequestURI(hdr.Target)
  203. if err != nil || !strings.HasPrefix(hdr.Target, "/") {
  204. c.write(frameKindResponse, frameHeader{ID: hdr.ID, Status: http.StatusBadRequest}, []byte("invalid tunnel target"))
  205. return
  206. }
  207. req, err := http.NewRequest(hdr.Method, hdr.Target, bytes.NewReader(body))
  208. if err != nil {
  209. c.write(frameKindResponse, frameHeader{ID: hdr.ID, Status: http.StatusBadRequest}, []byte(err.Error()))
  210. return
  211. }
  212. req.URL = u
  213. req.RequestURI = hdr.Target
  214. req.RemoteAddr = "tunnel"
  215. for k, vals := range hdr.Headers {
  216. for _, v := range vals {
  217. req.Header.Add(k, v)
  218. }
  219. }
  220. req.ContentLength = int64(len(body))
  221. rec := &responseRecorder{header: http.Header{}, status: http.StatusOK}
  222. handler.ServeHTTP(rec, req)
  223. c.write(frameKindResponse, frameHeader{
  224. ID: hdr.ID,
  225. Status: rec.status,
  226. Headers: map[string][]string(rec.header),
  227. }, rec.body.Bytes())
  228. }
  229. // readLoop pumps frames until the socket dies. Response frames complete
  230. // pending round trips; request frames are served through handler (nil when
  231. // this end never expects requests).
  232. func (c *tunnelConn) readLoop(handler http.Handler) {
  233. defer c.close()
  234. c.ws.SetReadDeadline(time.Now().Add(TunnelReadTimeout))
  235. c.ws.SetPongHandler(func(string) error {
  236. c.ws.SetReadDeadline(time.Now().Add(TunnelReadTimeout))
  237. return nil
  238. })
  239. c.ws.SetPingHandler(func(data string) error {
  240. c.ws.SetReadDeadline(time.Now().Add(TunnelReadTimeout))
  241. c.writeMu.Lock()
  242. defer c.writeMu.Unlock()
  243. return c.ws.WriteControl(websocket.PongMessage, []byte(data), time.Now().Add(10*time.Second))
  244. })
  245. for {
  246. mt, data, err := c.ws.ReadMessage()
  247. if err != nil {
  248. if !c.isClosed() && !websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
  249. logger.PrintAndLog("Cluster", "Tunnel read ended: "+err.Error(), nil)
  250. }
  251. return
  252. }
  253. c.ws.SetReadDeadline(time.Now().Add(TunnelReadTimeout))
  254. if mt != websocket.BinaryMessage {
  255. continue
  256. }
  257. kind, hdr, body, err := decodeFrame(data)
  258. if err != nil {
  259. continue
  260. }
  261. switch kind {
  262. case frameKindResponse:
  263. c.pmu.Lock()
  264. ch, ok := c.pending[hdr.ID]
  265. c.pmu.Unlock()
  266. if ok {
  267. ch <- &Response{Status: hdr.Status, Header: http.Header(hdr.Headers), Body: body}
  268. }
  269. case frameKindRequest:
  270. if handler != nil {
  271. go c.serveRequest(handler, hdr, body)
  272. }
  273. }
  274. }
  275. }
  276. // pingLoop keeps the socket alive through proxies until it closes.
  277. func (c *tunnelConn) pingLoop() {
  278. ticker := time.NewTicker(TunnelPingInterval)
  279. defer ticker.Stop()
  280. for {
  281. select {
  282. case <-c.closed:
  283. return
  284. case <-ticker.C:
  285. if err := c.ping(); err != nil {
  286. c.close()
  287. return
  288. }
  289. }
  290. }
  291. }
  292. // responseRecorder captures a handler's output for the response frame.
  293. type responseRecorder struct {
  294. header http.Header
  295. status int
  296. body bytes.Buffer
  297. wrote bool
  298. }
  299. func (r *responseRecorder) Header() http.Header { return r.header }
  300. func (r *responseRecorder) Write(b []byte) (int, error) {
  301. r.wrote = true
  302. return r.body.Write(b)
  303. }
  304. func (r *responseRecorder) WriteHeader(status int) {
  305. if !r.wrote {
  306. r.status = status
  307. }
  308. }
  309. /*
  310. Host side
  311. */
  312. // TunnelHub terminates tunnels from NAT-only nodes on a reachable node.
  313. type TunnelHub struct {
  314. Verifier *Verifier
  315. OnConnect func(nodeID string)
  316. OnDisconnect func(nodeID string)
  317. mu sync.RWMutex
  318. conns map[string]*tunnelConn
  319. upgrader websocket.Upgrader
  320. }
  321. // NewTunnelHub creates a hub whose incoming tunnels are authenticated by v.
  322. func NewTunnelHub(v *Verifier) *TunnelHub {
  323. return &TunnelHub{
  324. Verifier: v,
  325. conns: map[string]*tunnelConn{},
  326. upgrader: websocket.Upgrader{
  327. ReadBufferSize: 32 * 1024,
  328. WriteBufferSize: 32 * 1024,
  329. CheckOrigin: func(r *http.Request) bool { return true },
  330. },
  331. }
  332. }
  333. // HandleTunnel is the HTTP handler for TunnelPath. The upgrade request must be
  334. // signed by a cluster member.
  335. func (h *TunnelHub) HandleTunnel(w http.ResponseWriter, r *http.Request) {
  336. identity, _, err := h.Verifier.VerifyRequest(r)
  337. if err != nil {
  338. http.Error(w, err.Error(), http.StatusUnauthorized)
  339. return
  340. }
  341. ws, err := h.upgrader.Upgrade(w, r, nil)
  342. if err != nil {
  343. return
  344. }
  345. conn := newTunnelConn(ws)
  346. h.mu.Lock()
  347. if old, exists := h.conns[identity.NodeID]; exists {
  348. old.close()
  349. }
  350. h.conns[identity.NodeID] = conn
  351. h.mu.Unlock()
  352. logger.PrintAndLog("Cluster", "Tunnel established from node "+identity.NodeID, nil)
  353. if h.OnConnect != nil {
  354. h.OnConnect(identity.NodeID)
  355. }
  356. go conn.pingLoop()
  357. conn.readLoop(nil)
  358. h.mu.Lock()
  359. if cur, exists := h.conns[identity.NodeID]; exists && cur == conn {
  360. delete(h.conns, identity.NodeID)
  361. }
  362. h.mu.Unlock()
  363. logger.PrintAndLog("Cluster", "Tunnel from node "+identity.NodeID+" closed", nil)
  364. if h.OnDisconnect != nil {
  365. h.OnDisconnect(identity.NodeID)
  366. }
  367. }
  368. // Connected reports whether nodeID currently has a live tunnel here.
  369. func (h *TunnelHub) Connected(nodeID string) bool {
  370. h.mu.RLock()
  371. defer h.mu.RUnlock()
  372. c, ok := h.conns[nodeID]
  373. return ok && !c.isClosed()
  374. }
  375. // ConnectedNodes lists the node IDs with a live tunnel.
  376. func (h *TunnelHub) ConnectedNodes() []string {
  377. h.mu.RLock()
  378. defer h.mu.RUnlock()
  379. ids := []string{}
  380. for id, c := range h.conns {
  381. if !c.isClosed() {
  382. ids = append(ids, id)
  383. }
  384. }
  385. return ids
  386. }
  387. // Do forwards an already signed request to the tunnelled node.
  388. func (h *TunnelHub) Do(ctx context.Context, nodeID string, method string, target string, headers http.Header, body []byte) (*Response, error) {
  389. h.mu.RLock()
  390. c, ok := h.conns[nodeID]
  391. h.mu.RUnlock()
  392. if !ok || c.isClosed() {
  393. return nil, ErrTunnelNotConnected
  394. }
  395. return c.roundTrip(ctx, method, target, headers, body)
  396. }
  397. // Close drops every tunnel.
  398. func (h *TunnelHub) Close() {
  399. h.mu.Lock()
  400. defer h.mu.Unlock()
  401. for id, c := range h.conns {
  402. c.close()
  403. delete(h.conns, id)
  404. }
  405. }
  406. /*
  407. Client side
  408. */
  409. // TunnelHost describes the reachable peer a NAT-only node should tunnel through.
  410. type TunnelHost struct {
  411. NodeID string
  412. URL string
  413. }
  414. // TunnelClient keeps a NAT-only node attached to a reachable peer.
  415. type TunnelClient struct {
  416. Signer *Signer
  417. Handler http.Handler //Local ACN handler that serves the multiplexed requests
  418. //PickHost returns the peer to connect to. ok is false when no host is available yet.
  419. PickHost func() (TunnelHost, bool)
  420. InsecureTLS bool
  421. OnStateChange func(connected bool, host TunnelHost)
  422. mu sync.Mutex
  423. stop chan struct{}
  424. running bool
  425. connected bool
  426. host TunnelHost
  427. current *tunnelConn
  428. dialer func(host TunnelHost) (*websocket.Conn, error)
  429. }
  430. // Start begins the connect / serve / reconnect loop in the background.
  431. func (c *TunnelClient) Start() {
  432. c.mu.Lock()
  433. if c.running {
  434. c.mu.Unlock()
  435. return
  436. }
  437. c.running = true
  438. c.stop = make(chan struct{})
  439. stop := c.stop
  440. c.mu.Unlock()
  441. go c.loop(stop)
  442. }
  443. // Stop closes the current tunnel and halts reconnection.
  444. func (c *TunnelClient) Stop() {
  445. c.mu.Lock()
  446. if !c.running {
  447. c.mu.Unlock()
  448. return
  449. }
  450. c.running = false
  451. close(c.stop)
  452. cur := c.current
  453. c.mu.Unlock()
  454. if cur != nil {
  455. cur.close()
  456. }
  457. }
  458. // Reconnect drops the current tunnel so the loop picks a host again, used
  459. // after the preferred host changes.
  460. func (c *TunnelClient) Reconnect() {
  461. c.mu.Lock()
  462. cur := c.current
  463. c.mu.Unlock()
  464. if cur != nil {
  465. cur.close()
  466. }
  467. }
  468. // Status reports whether the tunnel is up and through which host.
  469. func (c *TunnelClient) Status() (bool, TunnelHost) {
  470. c.mu.Lock()
  471. defer c.mu.Unlock()
  472. return c.connected, c.host
  473. }
  474. func (c *TunnelClient) setState(connected bool, host TunnelHost, conn *tunnelConn) {
  475. c.mu.Lock()
  476. c.connected = connected
  477. c.host = host
  478. c.current = conn
  479. cb := c.OnStateChange
  480. c.mu.Unlock()
  481. if cb != nil {
  482. cb(connected, host)
  483. }
  484. }
  485. func (c *TunnelClient) dial(host TunnelHost) (*websocket.Conn, error) {
  486. if c.dialer != nil {
  487. return c.dialer(host)
  488. }
  489. wsURL, err := WebSocketURL(host.URL, TunnelPath)
  490. if err != nil {
  491. return nil, err
  492. }
  493. req, err := http.NewRequest(http.MethodGet, wsURL, nil)
  494. if err != nil {
  495. return nil, err
  496. }
  497. c.Signer.Sign(req, TunnelPath, []byte{})
  498. dialer := websocket.Dialer{
  499. HandshakeTimeout: 20 * time.Second,
  500. TLSClientConfig: tlsConfig(c.InsecureTLS),
  501. }
  502. ws, resp, err := dialer.Dial(wsURL, req.Header)
  503. if err != nil {
  504. if resp != nil && resp.Body != nil {
  505. msg, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
  506. resp.Body.Close()
  507. if len(msg) > 0 {
  508. return nil, errors.New(err.Error() + ": " + strings.TrimSpace(string(msg)))
  509. }
  510. }
  511. return nil, err
  512. }
  513. return ws, nil
  514. }
  515. func (c *TunnelClient) loop(stop chan struct{}) {
  516. backoff := 2 * time.Second
  517. for {
  518. select {
  519. case <-stop:
  520. return
  521. default:
  522. }
  523. host, ok := c.PickHost()
  524. if !ok {
  525. if !sleepOrStop(stop, 5*time.Second) {
  526. return
  527. }
  528. continue
  529. }
  530. ws, err := c.dial(host)
  531. if err != nil {
  532. logger.PrintAndLog("Cluster", "Tunnel to "+host.NodeID+" failed: "+err.Error(), nil)
  533. if !sleepOrStop(stop, backoff) {
  534. return
  535. }
  536. if backoff < 60*time.Second {
  537. backoff *= 2
  538. }
  539. continue
  540. }
  541. backoff = 2 * time.Second
  542. conn := newTunnelConn(ws)
  543. c.setState(true, host, conn)
  544. logger.PrintAndLog("Cluster", "Tunnel connected through node "+host.NodeID, nil)
  545. go conn.pingLoop()
  546. conn.readLoop(c.Handler) //blocks until the socket dies
  547. c.setState(false, host, nil)
  548. logger.PrintAndLog("Cluster", "Tunnel through node "+host.NodeID+" disconnected", nil)
  549. if !sleepOrStop(stop, backoff) {
  550. return
  551. }
  552. }
  553. }
  554. func sleepOrStop(stop chan struct{}, d time.Duration) bool {
  555. select {
  556. case <-stop:
  557. return false
  558. case <-time.After(d):
  559. return true
  560. }
  561. }
  562. // WebSocketURL converts a node base URL plus path into a ws:// or wss:// URL.
  563. func WebSocketURL(baseURL string, path string) (string, error) {
  564. u, err := url.Parse(strings.TrimRight(baseURL, "/"))
  565. if err != nil {
  566. return "", err
  567. }
  568. switch u.Scheme {
  569. case "https":
  570. u.Scheme = "wss"
  571. case "http":
  572. u.Scheme = "ws"
  573. case "ws", "wss":
  574. default:
  575. return "", errors.New("unsupported node URL scheme: " + u.Scheme)
  576. }
  577. u.Path = strings.TrimRight(u.Path, "/") + path
  578. u.RawQuery = ""
  579. return u.String(), nil
  580. }