correct varint encoding for the 1.5 UDP protocol

Encode values the way protobuf does rather than with Mumble's
tunnel-specific varint format.
The 1.5 native UDP protocol carries protobuf messages, so the two
encodings were being mixed and the server rejected our packets.

Handle the minimum signed 64-bit value without overflowing.
Negating math.MinInt64 wraps back to itself, which produced a
corrupt encoding instead of an error.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

gumble/gumble/varint/varint_test.go

@@ -1,6 +1,26 @@
 package varint // import "git.stormux.org/storm/barnard/gumble/gumble/varint"
 
-import "testing"
+import (
+	"math"
+	"testing"
+)
+
+// Regression: MinInt64 formerly caused unbounded recursive encoding, and a
+// caller-provided short buffer caused an index panic.
+func TestEncodeMinInt64AndShortBuffer(t *testing.T) {
+	buf := make([]byte, MaxVarintLen)
+	n := Encode(buf, math.MinInt64)
+	if n != MaxVarintLen {
+		t.Fatalf("length = %d, want %d", n, MaxVarintLen)
+	}
+	got, consumed := Decode(buf[:n])
+	if consumed != n || got != math.MinInt64 {
+		t.Fatalf("decoded (%d, %d)", got, consumed)
+	}
+	if n := Encode(make([]byte, 1), 128); n != 0 {
+		t.Fatalf("short buffer returned %d", n)
+	}
+}
 
 func TestRange(t *testing.T) {
 

gumble/gumble/varint/write.go

@@ -9,56 +9,59 @@ import (
 // number.
 const MaxVarintLen = 10
 
-// Encode encodes the given value to varint format.
+// Encode encodes value in the Mumble varint format. It returns zero when b is
+// too small, rather than panicking on a caller-provided short buffer.
 func Encode(b []byte, value int64) int {
-	// 111111xx Byte-inverted negative two bit number (~xx)
+	var encoded [MaxVarintLen]byte
+	n := encode(encoded[:], value)
+	if n == 0 || len(b) < n {
+		return 0
+	}
+	copy(b, encoded[:n])
+	return n
+}
+
+func encode(b []byte, value int64) int {
 	if value <= -1 && value >= -4 {
 		b[0] = 0xFC | byte(^value&0xFF)
 		return 1
 	}
-	// 111110__ + varint Negative recursive varint
 	if value < 0 {
 		b[0] = 0xF8
-		return 1 + Encode(b[1:], -value)
+		// -math.MinInt64 overflows. The decoder intentionally interprets the
+		// following signed 64-bit payload as MinInt64 and negates it modulo 2^64.
+		if value == math.MinInt64 {
+			b[1] = 0xF4
+			binary.BigEndian.PutUint64(b[2:], uint64(value))
+			return 10
+		}
+		return 1 + encode(b[1:], -value)
 	}
-	// 0xxxxxxx 7-bit positive number
 	if value <= 0x7F {
 		b[0] = byte(value)
 		return 1
 	}
-	// 10xxxxxx + 1 byte 14-bit positive number
 	if value <= 0x3FFF {
-		b[0] = byte(((value >> 8) & 0x3F) | 0x80)
+		b[0] = byte(value>>8)&0x3F | 0x80
-		b[1] = byte(value & 0xFF)
+		b[1] = byte(value)
 		return 2
 	}
-	// 110xxxxx + 2 bytes 21-bit positive number
 	if value <= 0x1FFFFF {
-		b[0] = byte((value>>16)&0x1F | 0xC0)
+		b[0] = byte(value>>16)&0x1F | 0xC0
-		b[1] = byte((value >> 8) & 0xFF)
+		b[1], b[2] = byte(value>>8), byte(value)
-		b[2] = byte(value & 0xFF)
 		return 3
 	}
-	// 1110xxxx + 3 bytes 28-bit positive number
 	if value <= 0xFFFFFFF {
-		b[0] = byte((value>>24)&0xF | 0xE0)
+		b[0] = byte(value>>24)&0x0F | 0xE0
-		b[1] = byte((value >> 16) & 0xFF)
+		b[1], b[2], b[3] = byte(value>>16), byte(value>>8), byte(value)
-		b[2] = byte((value >> 8) & 0xFF)
-		b[3] = byte(value & 0xFF)
 		return 4
 	}
-	// 111100__ + int (32-bit) 32-bit positive number
 	if value <= math.MaxInt32 {
 		b[0] = 0xF0
 		binary.BigEndian.PutUint32(b[1:], uint32(value))
 		return 5
 	}
-	// 111101__ + long (64-bit) 64-bit number
+	b[0] = 0xF4
-	if value <= math.MaxInt64 {
+	binary.BigEndian.PutUint64(b[1:], uint64(value))
-		b[0] = 0xF4
+	return 9
-		binary.BigEndian.PutUint64(b[1:], uint64(value))
-		return 9
-	}
-
-	return 0
 }