start migrating the TunAdapter to the actor model

src/tuntap/conn.go

@@ -113,8 +113,7 @@ func (s *tunConn) _read(bs []byte) (err error) {
 		util.PutBytes(bs)
 		return
 	}
-	// FIXME this send can block if the tuntap isn't running, which isn't really safe...
+	s.tun.writer.writeFrom(s, bs)
-	s.tun.send <- bs
 	s.stillAlive()
 	return
 }
@@ -208,7 +207,7 @@ func (s *tunConn) _write(bs []byte) (err error) {
 					Data: bs[:900],
 				}
 				if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil {
-					s.tun.send <- packet
+					s.tun.writer.writeFrom(s, packet)
 				}
 			} else {
 				if e.Closed() {

src/tuntap/iface.go

@@ -9,48 +9,60 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
-func (tun *TunAdapter) writer() error {
+type tunWriter struct {
-	var w int
+	phony.Inbox
+	tun *TunAdapter
+}
+
+func (w *tunWriter) writeFrom(from phony.Actor, b []byte) {
+	w.RecvFrom(from, func() {
+		w._write(b)
+	})
+}
+
+// write is pretty loose with the memory safety rules, e.g. it assumes it can read w.tun.iface.IsTap() safely
+func (w *tunWriter) _write(b []byte) {
+	var written int
 	var err error
-	for {
-		b := <-tun.send
 	n := len(b)
 	if n == 0 {
-			continue
+		return
 	}
-		if tun.iface.IsTAP() {
+	if w.tun.iface.IsTAP() {
 		sendndp := func(dstAddr address.Address) {
-				neigh, known := tun.icmpv6.getNeighbor(dstAddr)
+			neigh, known := w.tun.icmpv6.getNeighbor(dstAddr)
 			known = known && (time.Since(neigh.lastsolicitation).Seconds() < 30)
 			if !known {
-					tun.icmpv6.Solicit(dstAddr)
+				w.tun.icmpv6.Solicit(dstAddr)
 			}
 		}
 		peermac := net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
 		var dstAddr address.Address
 		var peerknown bool
 		if b[0]&0xf0 == 0x40 {
-				dstAddr = tun.addr
+			dstAddr = w.tun.addr
 		} else if b[0]&0xf0 == 0x60 {
-				if !bytes.Equal(tun.addr[:16], dstAddr[:16]) && !bytes.Equal(tun.subnet[:8], dstAddr[:8]) {
+			if !bytes.Equal(w.tun.addr[:16], dstAddr[:16]) && !bytes.Equal(w.tun.subnet[:8], dstAddr[:8]) {
-					dstAddr = tun.addr
+				dstAddr = w.tun.addr
 			}
 		}
-			if neighbor, ok := tun.icmpv6.getNeighbor(dstAddr); ok && neighbor.learned {
+		if neighbor, ok := w.tun.icmpv6.getNeighbor(dstAddr); ok && neighbor.learned {
 			// If we've learned the MAC of a 300::/7 address, for example, or a CKR
 			// address, use the MAC address of that
 			peermac = neighbor.mac
 			peerknown = true
-			} else if neighbor, ok := tun.icmpv6.getNeighbor(tun.addr); ok && neighbor.learned {
+		} else if neighbor, ok := w.tun.icmpv6.getNeighbor(w.tun.addr); ok && neighbor.learned {
 			// Otherwise send directly to the MAC address of the host if that's
 			// known instead
 			peermac = neighbor.mac
 			peerknown = true
 		} else {
 			// Nothing has been discovered, try to discover the destination
-				sendndp(tun.addr)
+			sendndp(w.tun.addr)
 		}
 		if peerknown {
 			var proto ethernet.Ethertype
@@ -63,38 +75,67 @@ func (tun *TunAdapter) writer() error {
 			var frame ethernet.Frame
 			frame.Prepare(
 				peermac[:6],            // Destination MAC address
-					tun.icmpv6.mymac[:6], // Source MAC address
+				w.tun.icmpv6.mymac[:6], // Source MAC address
 				ethernet.NotTagged,     // VLAN tagging
 				proto,                  // Ethertype
 				len(b))                 // Payload length
 			copy(frame[tun_ETHER_HEADER_LENGTH:], b[:n])
 			n += tun_ETHER_HEADER_LENGTH
-				w, err = tun.iface.Write(frame[:n])
+			written, err = w.tun.iface.Write(frame[:n])
 		} else {
-				tun.log.Errorln("TUN/TAP iface write error: no peer MAC known for", net.IP(dstAddr[:]).String(), "- dropping packet")
+			w.tun.log.Errorln("TUN/TAP iface write error: no peer MAC known for", net.IP(dstAddr[:]).String(), "- dropping packet")
 		}
 	} else {
-			w, err = tun.iface.Write(b[:n])
+		written, err = w.tun.iface.Write(b[:n])
 		util.PutBytes(b)
 	}
 	if err != nil {
-			if !tun.isOpen {
+		w.tun.mutex.Lock()
-				return err
+		open := w.tun.isOpen
+		w.tun.mutex.Unlock()
+		if !open {
+			return
 		}
-			tun.log.Errorln("TUN/TAP iface write error:", err)
+		w.tun.log.Errorln("TUN/TAP iface write error:", err)
-			continue
-		}
-		if w != n {
-			tun.log.Errorln("TUN/TAP iface write mismatch:", w, "bytes written vs", n, "bytes given")
-			continue
 	}
+	if written != n {
+		w.tun.log.Errorln("TUN/TAP iface write mismatch:", written, "bytes written vs", n, "bytes given")
 	}
 }
 
-// Run in a separate goroutine by the reader
+type tunReader struct {
-// Does all of the per-packet ICMP checks, passes packets to the right Conn worker
+	phony.Inbox
-func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
+	tun *TunAdapter
-	for recvd := range ch {
+}
+
+func (r *tunReader) _read() {
+	// Get a slice to store the packet in
+	recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
+	// Wait for a packet to be delivered to us through the TUN/TAP adapter
+	n, err := r.tun.iface.Read(recvd)
+	if n == 0 {
+		util.PutBytes(recvd)
+	} else {
+		r.tun.handlePacketFrom(r, recvd[:n], err)
+	}
+	if err == nil {
+		// Now read again
+		r.RecvFrom(nil, r._read)
+	}
+}
+
+func (tun *TunAdapter) handlePacketFrom(from phony.Actor, packet []byte, err error) {
+	tun.RecvFrom(from, func() {
+		tun._handlePacket(packet, err)
+	})
+}
+
+// does the work of reading a packet and sending it to the correct tunConn
+func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
+	if err != nil {
+		tun.log.Errorln("TUN/TAP iface read error:", err)
+		return
+	}
 	// If it's a TAP adapter, update the buffer slice so that we no longer
 	// include the ethernet headers
 	offset := 0
@@ -103,7 +144,7 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 		offset = tun_ETHER_HEADER_LENGTH
 		// Check first of all that we can go beyond the ethernet headers
 		if len(recvd) <= offset {
-				continue
+			return
 		}
 	}
 	// Offset the buffer from now on so that we can ignore ethernet frames if
@@ -117,7 +158,7 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 		if err := tun.icmpv6.ParsePacket(recvd); err == nil {
 			// We acted on the packet in the ICMPv6 module so don't forward or do
 			// anything else with it
-				continue
+			return
 		}
 	}
 	if offset != 0 {
@@ -136,11 +177,11 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 	if bs[0]&0xf0 == 0x60 {
 		// Check if we have a fully-sized IPv6 header
 		if len(bs) < 40 {
-				continue
+			return
 		}
 		// Check the packet size
 		if n-tun_IPv6_HEADER_LENGTH != 256*int(bs[4])+int(bs[5]) {
-				continue
+			return
 		}
 		// IPv6 address
 		addrlen = 16
@@ -149,11 +190,11 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 	} else if bs[0]&0xf0 == 0x40 {
 		// Check if we have a fully-sized IPv4 header
 		if len(bs) < 20 {
-				continue
+			return
 		}
 		// Check the packet size
 		if n != 256*int(bs[2])+int(bs[3]) {
-				continue
+			return
 		}
 		// IPv4 address
 		addrlen = 4
@@ -161,7 +202,7 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 	} else {
 		// Unknown address length or protocol, so drop the packet and ignore it
 		tun.log.Traceln("Unknown packet type, dropping")
-			continue
+		return
 	}
 	if tun.ckr.isEnabled() {
 		if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
@@ -176,7 +217,7 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 	}
 	if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
 		// Couldn't find this node's ygg IP
-			continue
+		return
 	}
 	// Do we have an active connection for this node address?
 	var dstNodeID, dstNodeIDMask *crypto.NodeID
@@ -237,36 +278,10 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
 			}
 		}()
 		// While the dial is going on we can't do much else
-			// continuing this iteration - skip to the next one
+		return
-			continue
 	}
 	// If we have a connection now, try writing to it
 	if isIn && session != nil {
-			<-session.SyncExec(func() { session._write(bs) })
+		session.RecvFrom(tun, func() { session._write(bs) })
-		}
-	}
-}
-
-func (tun *TunAdapter) reader() error {
-	toWorker := make(chan []byte, 32)
-	defer close(toWorker)
-	go tun.readerPacketHandler(toWorker)
-	for {
-		// Get a slice to store the packet in
-		recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
-		// Wait for a packet to be delivered to us through the TUN/TAP adapter
-		n, err := tun.iface.Read(recvd)
-		if err != nil {
-			if !tun.isOpen {
-				return err
-			}
-			panic(err)
-		}
-		if n == 0 {
-			util.PutBytes(recvd)
-			continue
-		}
-		// Send the packet to the worker
-		toWorker <- recvd[:n]
 	}
 }

src/tuntap/tun.go

@@ -15,6 +15,7 @@ import (
 	"net"
 	"sync"
 
+	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
 	"github.com/yggdrasil-network/water"
 
@@ -33,6 +34,8 @@ const tun_ETHER_HEADER_LENGTH = 14
 // you should pass this object to the yggdrasil.SetRouterAdapter() function
 // before calling yggdrasil.Start().
 type TunAdapter struct {
+	writer       tunWriter
+	reader       tunReader
 	config       *config.NodeState
 	log          *log.Logger
 	reconfigure  chan chan error
@@ -44,7 +47,7 @@ type TunAdapter struct {
 	icmpv6       ICMPv6
 	mtu          int
 	iface        *water.Interface
-	send         chan []byte
+	phony.Inbox               // Currently only used for _handlePacket from the reader, TODO: all the stuff that currently needs a mutex below
 	mutex        sync.RWMutex // Protects the below
 	addrToConn   map[address.Address]*tunConn
 	subnetToConn map[address.Subnet]*tunConn
@@ -114,6 +117,8 @@ func (tun *TunAdapter) Init(config *config.NodeState, log *log.Logger, listener
 	tun.addrToConn = make(map[address.Address]*tunConn)
 	tun.subnetToConn = make(map[address.Subnet]*tunConn)
 	tun.dials = make(map[crypto.NodeID][][]byte)
+	tun.writer.tun = tun
+	tun.reader.tun = tun
 }
 
 // Start the setup process for the TUN/TAP adapter. If successful, starts the
@@ -147,7 +152,6 @@ func (tun *TunAdapter) Start() error {
 	}
 	tun.mutex.Lock()
 	tun.isOpen = true
-	tun.send = make(chan []byte, 32) // TODO: is this a sensible value?
 	tun.reconfigure = make(chan chan error)
 	tun.mutex.Unlock()
 	go func() {
@@ -157,8 +161,7 @@ func (tun *TunAdapter) Start() error {
 		}
 	}()
 	go tun.handler()
-	go tun.reader()
+	tun.reader.RecvFrom(nil, tun.reader._read) // Start the reader
-	go tun.writer()
 	tun.icmpv6.Init(tun)
 	if iftapmode {
 		go tun.icmpv6.Solicit(tun.addr)