Fix HID transfers, harden the tunnel, add E2E crypto and direct peers

The HID failure came down to the endpoint type map being indexed by endpoint number without the direction bit. A composite device can have endpoint 1 as both interrupt IN (0x81) and bulk OUT (0x01); the last one read won, so interrupt URBs were submitted as bulk and the kernel rejected them. The device attached and stayed silent. Endpoint data now comes from the raw descriptors read from /dev/bus/usb rather than sysfs, which only ever exposes the active alternate setting — a webcam's isochronous endpoints are invisible there because they only exist after SET_INTERFACE. Two sysfs parsing bugs fell out of that too: the numeric endpoint attributes are hex without a prefix (wMaxPacketSize "0040" was read as 40, not 64), and bInterval was never read at all. Reliability: three places could freeze the whole process. The share path fed io.Pipe from the WebSocket read loop, so one slow USB transfer stalled every tunnel and the keepalives with them. The relay wrote to client sockets while holding the hub lock, so one peer that stopped reading blocked routing and registration for everyone. Control transfers ran inline in the protocol loop behind a 5s timeout. Also fixed: a use-after- free where a discarded URB's memory could be collected while the kernel still owned it, a reap loop that spun at 100% CPU on ioctl errors, a missing attach timeout, a double close(done) panic, and Hash[:8] in the relay's log line, which let a client with a short hash take the server down. Adds mode "both", so one client can offer and consume devices at once. The tunnel and client-left callbacks became multicast for it: as plain fields the second manager to register silently unhooked the first. Tunnel traffic is now AES-256-GCM end to end, on the relay path as well as directly. The key is derived from the three tokens, not from the group hash — the relay is told the hash, so a key derived from it would protect nothing from the one party in the middle. Group IDs are unchanged, so existing setups keep working; only clients configured without the tokens drop to unencrypted, relay-only operation. Peers now try to connect directly, with the relay supplying the public address neither side can determine for itself. Candidates are raced because an unreachable address hangs until timeout rather than refusing. Falling back to the relay is not an error. Platform reach: cross-compiled targets for ARM, MIPS and RISC-V (the Linux client needed no code changes — usbdevfs is not architecture specific), multi-arch Docker images, an Android bridge that accepts devices over SCM_RIGHTS because apps cannot open /dev/bus/usb, and macOS builds via system_profiler enumeration. Adds a Windows KMDF filter driver under driver/windows with its Go side. UNTESTED: it has never been compiled or run, needs the WDK to build and an EV certificate to distribute. Treat it as a starting point. Adds "usb-client diag": says per machine whether sharing and using are possible, what stands in the way, and what fixes it. Reports can be uploaded to a relay to get them off machines that are awkward to copy from. 96 tests, all green under -race. Builds for linux, windows and darwin on amd64 and arm64. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-11 22:02:04 +02:00
parent 54178dce75
commit 9ed473a965
95 changed files with 12181 additions and 892 deletions
@@ -3,36 +3,101 @@ package relay
 import (
 	"encoding/json"
 	"log"
+	"net"
+	"strconv"
 	"sync"

 	"github.com/duffy/usb-server/internal/protocol"
 	"github.com/gorilla/websocket"
 )

+// sendQueueDepth bounds per-client outgoing backlog. A client that falls this
+// far behind is not going to catch up, and buffering more would let one stuck
+// peer consume the relay's memory.
+const sendQueueDepth = 256
+
+// outMsg is one queued WebSocket frame.
+type outMsg struct {
+	typ  int // websocket.TextMessage or websocket.BinaryMessage
+	data []byte
+}
+
 // Client represents a connected WebSocket client
 type Client struct {
 	ID   string
 	Hash string
-	Mode string // "share" or "use"
+	Mode string // "share", "use" or "both"
 	Name string
 	Conn *websocket.Conn
-	Send chan []byte // buffered channel for outgoing messages

-	mu sync.Mutex
+	// DirectPort is the port this client accepts direct tunnel connections on,
+	// 0 if it accepts none.
+	DirectPort int
+
+	// PublicIP is the source address the relay sees this client connect from.
+	// Peers cannot determine their own public address, so the relay supplies
+	// it when passing on a grant — that is the whole reason it is involved in
+	// setting up connections that then bypass it.
+	PublicIP string
+
+	// Send carries outgoing frames to this client's write pump. All writes go
+	// through it: writing to the socket directly from another client's read
+	// loop would block that peer — and, because the hub held its lock across
+	// the write, every other client with it.
+	Send chan outMsg
+
+	closeOnce sync.Once
+	dead      chan struct{}
 }

-// WriteJSON sends a JSON message to the client
-func (c *Client) WriteJSON(v interface{}) error {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.Conn.WriteJSON(v)
+// newClient creates a client with its outgoing queue ready.
+func newClient(id, hash, mode, name string, conn *websocket.Conn) *Client {
+	return &Client{
+		ID:   id,
+		Hash: hash,
+		Mode: mode,
+		Name: name,
+		Conn: conn,
+		Send: make(chan outMsg, sendQueueDepth),
+		dead: make(chan struct{}),
+	}
 }

-// WriteBinary sends a binary message to the client
-func (c *Client) WriteBinary(data []byte) error {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.Conn.WriteMessage(websocket.BinaryMessage, data)
+// enqueue queues a frame without blocking.
+// It reports false when the client's queue is full or it is already gone; the
+// caller should treat that as a disconnect rather than retrying.
+func (c *Client) enqueue(typ int, data []byte) bool {
+	select {
+	case <-c.dead:
+		return false
+	default:
+	}
+
+	select {
+	case c.Send <- outMsg{typ: typ, data: data}:
+		return true
+	case <-c.dead:
+		return false
+	default:
+		log.Printf("[hub] send queue full for %s (%s), dropping client",
+			protocol.ShortID(c.ID), c.Name)
+		c.kill()
+		return false
+	}
+}
+
+// kill marks the client dead and wakes its write pump. Idempotent.
+func (c *Client) kill() {
+	c.closeOnce.Do(func() { close(c.dead) })
+}
+
+// enqueueJSON marshals and queues a JSON control message.
+func (c *Client) enqueueJSON(v interface{}) bool {
+	data, err := json.Marshal(v)
+	if err != nil {
+		return false
+	}
+	return c.enqueue(websocket.TextMessage, data)
 }

 // Hub manages all connected clients and routes messages between them
@@ -58,39 +123,80 @@ func NewHub() *Hub {
 	}
 }

+// peers returns a snapshot of the clients in a hash group, excluding one ID.
+//
+// Taking a snapshot and releasing the lock before doing anything with the
+// clients is deliberate: holding the hub lock across a send is what let a
+// single slow peer stall registration and routing for everyone.
+func (h *Hub) peers(hash, excludeID string) []*Client {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	group := h.groups[hash]
+	result := make([]*Client, 0, len(group))
+	for _, c := range group {
+		if c.ID != excludeID {
+			result = append(result, c)
+		}
+	}
+	return result
+}
+
+// peer looks up a single client in a hash group.
+func (h *Hub) peer(hash, clientID string) *Client {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	group := h.groups[hash]
+	if group == nil {
+		return nil
+	}
+	return group[clientID]
+}
+
 // Register adds a client to its hash group
 func (h *Hub) Register(client *Client) {
 	h.mu.Lock()
-	defer h.mu.Unlock()
-
 	if h.groups[client.Hash] == nil {
 		h.groups[client.Hash] = make(map[string]*Client)
 	}
+	// A reconnecting client reuses its ID; drop the stale entry so its
+	// write pump exits instead of lingering with a dead socket.
+	if old, exists := h.groups[client.Hash][client.ID]; exists && old != client {
+		old.kill()
+	}
 	h.groups[client.Hash][client.ID] = client
+	h.mu.Unlock()

-	log.Printf("[hub] client registered: id=%s hash=%s..%s mode=%s name=%s",
-		client.ID, client.Hash[:8], client.Hash[len(client.Hash)-4:], client.Mode, client.Name)
+	log.Printf("[hub] client registered: id=%s hash=%s mode=%s name=%s",
+		protocol.ShortID(client.ID), protocol.ShortID(client.Hash), client.Mode, client.Name)

 	// Notify other clients in the group
-	h.broadcastToGroup(client.Hash, client.ID, &protocol.ClientJoined{
+	joined := &protocol.ClientJoined{
 		Type:     protocol.MsgClientJoined,
 		ClientID: client.ID,
 		Mode:     client.Mode,
 		Name:     client.Name,
-	})
+	}
+	for _, peer := range h.peers(client.Hash, client.ID) {
+		peer.enqueueJSON(joined)
+	}
 }

 // Unregister removes a client and cleans up its tunnels
 func (h *Hub) Unregister(client *Client) {
 	h.mu.Lock()
-	defer h.mu.Unlock()
-
 	group := h.groups[client.Hash]
 	if group == nil {
+		h.mu.Unlock()
 		return
 	}

-	delete(group, client.ID)
+	// Only remove this exact client: a reconnect may already have installed a
+	// newer connection under the same ID.
+	if group[client.ID] == client {
+		delete(group, client.ID)
+	}
 	if len(group) == 0 {
 		delete(h.groups, client.Hash)
 	}
@@ -101,21 +207,26 @@ func (h *Hub) Unregister(client *Client) {
 			delete(h.tunnels, tid)
 		}
 	}
+	h.mu.Unlock()

-	log.Printf("[hub] client unregistered: id=%s name=%s", client.ID, client.Name)
+	client.kill()

-	// Notify others
-	h.broadcastToGroup(client.Hash, client.ID, &protocol.ClientLeft{
+	log.Printf("[hub] client unregistered: id=%s name=%s", protocol.ShortID(client.ID), client.Name)
+
+	left := &protocol.ClientLeft{
 		Type:     protocol.MsgClientLeft,
 		ClientID: client.ID,
-	})
+	}
+	for _, peer := range h.peers(client.Hash, client.ID) {
+		peer.enqueueJSON(left)
+	}
 }

 // HandleTextMessage processes a JSON control message
 func (h *Hub) HandleTextMessage(sender *Client, data []byte) {
 	var env protocol.Envelope
 	if err := json.Unmarshal(data, &env); err != nil {
-		log.Printf("[hub] invalid message from %s: %v", sender.ID, err)
+		log.Printf("[hub] invalid message from %s: %v", protocol.ShortID(sender.ID), err)
 		return
 	}

@@ -135,9 +246,9 @@ func (h *Hub) HandleTextMessage(sender *Client, data []byte) {
 	case protocol.MsgDeviceReleased:
 		h.handleDeviceReleased(sender, data)
 	case protocol.MsgPing:
-		sender.WriteJSON(&protocol.Pong{Type: protocol.MsgPong})
+		sender.enqueueJSON(&protocol.Pong{Type: protocol.MsgPong})
 	default:
-		log.Printf("[hub] unknown message type from %s: %s", sender.ID, env.Type)
+		log.Printf("[hub] unknown message type from %s: %s", protocol.ShortID(sender.ID), env.Type)
 	}
 }

@@ -159,40 +270,32 @@ func (h *Hub) HandleBinaryMessage(sender *Client, data []byte) {

 	// Forward to the other end of the tunnel
 	var targetID string
-	if sender.ID == tunnel.ShareClient {
+	switch sender.ID {
+	case tunnel.ShareClient:
 		targetID = tunnel.UseClient
-	} else if sender.ID == tunnel.UseClient {
+	case tunnel.UseClient:
 		targetID = tunnel.ShareClient
-	} else {
+	default:
 		return
 	}

-	h.mu.RLock()
-	group := h.groups[sender.Hash]
-	if group != nil {
-		if target := group[targetID]; target != nil {
-			target.WriteBinary(data)
-		}
+	if target := h.peer(sender.Hash, targetID); target != nil {
+		target.enqueue(websocket.BinaryMessage, data)
 	}
-	h.mu.RUnlock()
 }

-// handleDeviceList broadcasts device list from share client to all use clients
+// handleDeviceList broadcasts a device list to every client in the group that
+// can consume devices.
 func (h *Hub) handleDeviceList(sender *Client, data []byte) {
-	if sender.Mode != protocol.ModeShare {
+	if !protocol.CanShare(sender.Mode) {
 		return
 	}

-	h.mu.RLock()
-	group := h.groups[sender.Hash]
-	for _, client := range group {
-		if client.ID != sender.ID && client.Mode == protocol.ModeUse {
-			client.mu.Lock()
-			client.Conn.WriteMessage(websocket.TextMessage, data)
-			client.mu.Unlock()
+	for _, client := range h.peers(sender.Hash, sender.ID) {
+		if protocol.CanUse(client.Mode) {
+			client.enqueue(websocket.TextMessage, data)
 		}
 	}
-	h.mu.RUnlock()
 }

 // handleRequestDevice forwards a device request to the target share client
@@ -202,22 +305,19 @@ func (h *Hub) handleRequestDevice(sender *Client, data []byte) {
 		return
 	}

-	h.mu.RLock()
-	group := h.groups[sender.Hash]
-	if group != nil {
-		if target := group[msg.TargetClient]; target != nil && target.Mode == protocol.ModeShare {
-			// Add the sender's ID so the share client knows who's requesting
-			enriched := map[string]interface{}{
-				"type":          protocol.MsgRequestDevice,
-				"target_client": msg.TargetClient,
-				"bus_id":        msg.BusID,
-				"request_id":    msg.RequestID,
-				"from_client":   sender.ID,
-			}
-			target.WriteJSON(enriched)
-		}
+	target := h.peer(sender.Hash, msg.TargetClient)
+	if target == nil || !protocol.CanShare(target.Mode) {
+		return
 	}
-	h.mu.RUnlock()
+
+	// Add the sender's ID so the share client knows who's requesting
+	target.enqueueJSON(map[string]interface{}{
+		"type":          protocol.MsgRequestDevice,
+		"target_client": msg.TargetClient,
+		"bus_id":        msg.BusID,
+		"request_id":    msg.RequestID,
+		"from_client":   sender.ID,
+	})
 }

 // handleDeviceGranted registers the tunnel and forwards to the requesting client
@@ -229,8 +329,10 @@ func (h *Hub) handleDeviceGranted(sender *Client, data []byte) {
 	if err := json.Unmarshal(data, &granted); err != nil {
 		return
 	}
+	if granted.TunnelID == "" {
+		return
+	}

-	// Register tunnel
 	h.mu.Lock()
 	h.tunnels[granted.TunnelID] = &Tunnel{
 		ID:          granted.TunnelID,
@@ -241,19 +343,44 @@ func (h *Hub) handleDeviceGranted(sender *Client, data []byte) {
 	h.mu.Unlock()

 	log.Printf("[hub] tunnel created: %s (share=%s, use=%s, device=%s)",
-		granted.TunnelID, sender.ID, granted.TargetClient, granted.BusID)
+		granted.TunnelID, protocol.ShortID(sender.ID), protocol.ShortID(granted.TargetClient), granted.BusID)

-	// Forward to use client
-	h.mu.RLock()
-	group := h.groups[sender.Hash]
-	if group != nil {
-		if target := group[granted.TargetClient]; target != nil {
-			target.mu.Lock()
-			target.Conn.WriteMessage(websocket.TextMessage, data)
-			target.mu.Unlock()
+	target := h.peer(sender.Hash, granted.TargetClient)
+	if target == nil {
+		return
+	}
+
+	// Add the address we see the granting client at. It cannot know its own
+	// public address, and this is what lets the two peers connect directly
+	// across NAT and take their USB traffic off this relay entirely.
+	out := data
+	if extra := publicEndpoint(sender); extra != "" {
+		granted.Endpoints = appendUnique(granted.Endpoints, extra)
+		if reencoded, err := json.Marshal(granted); err == nil {
+			out = reencoded
 		}
 	}
-	h.mu.RUnlock()
+
+	target.enqueue(websocket.TextMessage, out)
+}
+
+// publicEndpoint builds the host:port at which a client's direct listener
+// should be reachable from outside, or "" if it accepts no direct connections.
+func publicEndpoint(c *Client) string {
+	if c.DirectPort == 0 || c.PublicIP == "" {
+		return ""
+	}
+	return net.JoinHostPort(c.PublicIP, strconv.Itoa(c.DirectPort))
+}
+
+// appendUnique adds an entry unless it is already present.
+func appendUnique(list []string, item string) []string {
+	for _, existing := range list {
+		if existing == item {
+			return list
+		}
+	}
+	return append(list, item)
 }

 // handleDeviceDenied forwards denial to the requesting client
@@ -266,16 +393,9 @@ func (h *Hub) handleDeviceDenied(sender *Client, data []byte) {
 		return
 	}

-	h.mu.RLock()
-	group := h.groups[sender.Hash]
-	if group != nil {
-		if target := group[denied.TargetClient]; target != nil {
-			target.mu.Lock()
-			target.Conn.WriteMessage(websocket.TextMessage, data)
-			target.mu.Unlock()
-		}
+	if target := h.peer(sender.Hash, denied.TargetClient); target != nil {
+		target.enqueue(websocket.TextMessage, data)
 	}
-	h.mu.RUnlock()
 }

 // handleReleaseDevice forwards a release to the share client
@@ -296,21 +416,14 @@ func (h *Hub) handleReleaseDevice(sender *Client, data []byte) {
 	}
 	h.mu.Unlock()

-	// Forward to share client
-	h.mu.RLock()
-	group := h.groups[sender.Hash]
-	if group != nil {
-		if target := group[msg.TargetClient]; target != nil {
-			enriched := map[string]interface{}{
-				"type":         protocol.MsgReleaseDevice,
-				"target_client": msg.TargetClient,
-				"bus_id":       msg.BusID,
-				"from_client":  sender.ID,
-			}
-			target.WriteJSON(enriched)
-		}
+	if target := h.peer(sender.Hash, msg.TargetClient); target != nil {
+		target.enqueueJSON(map[string]interface{}{
+			"type":          protocol.MsgReleaseDevice,
+			"target_client": msg.TargetClient,
+			"bus_id":        msg.BusID,
+			"from_client":   sender.ID,
+		})
 	}
-	h.mu.RUnlock()
 }

 // handleForceRelease forwards a force-release request to the target share client
@@ -331,43 +444,36 @@ func (h *Hub) handleForceRelease(sender *Client, data []byte) {
 	}
 	h.mu.Unlock()

-	// Forward to share client
-	h.mu.RLock()
-	group := h.groups[sender.Hash]
-	if group != nil {
-		if target := group[msg.TargetClient]; target != nil && target.Mode == protocol.ModeShare {
-			enriched := map[string]interface{}{
-				"type":          protocol.MsgForceRelease,
-				"target_client": msg.TargetClient,
-				"bus_id":        msg.BusID,
-				"from_client":   sender.ID,
-			}
-			target.WriteJSON(enriched)
-		}
+	target := h.peer(sender.Hash, msg.TargetClient)
+	if target == nil || !protocol.CanShare(target.Mode) {
+		return
 	}
-	h.mu.RUnlock()
+
+	target.enqueueJSON(map[string]interface{}{
+		"type":          protocol.MsgForceRelease,
+		"target_client": msg.TargetClient,
+		"bus_id":        msg.BusID,
+		"from_client":   sender.ID,
+	})
 }

 // handleDeviceReleased broadcasts device released notification
 func (h *Hub) handleDeviceReleased(sender *Client, data []byte) {
-	h.mu.RLock()
-	group := h.groups[sender.Hash]
-	for _, client := range group {
-		if client.ID != sender.ID && client.Mode == protocol.ModeUse {
-			client.mu.Lock()
-			client.Conn.WriteMessage(websocket.TextMessage, data)
-			client.mu.Unlock()
+	for _, client := range h.peers(sender.Hash, sender.ID) {
+		if protocol.CanUse(client.Mode) {
+			client.enqueue(websocket.TextMessage, data)
 		}
 	}
-	h.mu.RUnlock()
 }

-// broadcastToGroup sends a message to all clients in a hash group except the sender
-func (h *Hub) broadcastToGroup(hash, excludeID string, msg interface{}) {
-	group := h.groups[hash]
-	for _, client := range group {
-		if client.ID != excludeID {
-			client.WriteJSON(msg)
-		}
+// GroupStats reports the number of clients per hash group, for diagnostics.
+func (h *Hub) GroupStats() map[string]int {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	stats := make(map[string]int, len(h.groups))
+	for hash, group := range h.groups {
+		stats[protocol.ShortID(hash)] = len(group)
 	}
+	return stats
 }