diff --git a/barnard.go b/barnard.go
index 518a604..9f3604a 100644
--- a/barnard.go
+++ b/barnard.go
@@ -53,6 +53,7 @@ type Barnard struct {
 
 	// Added for channel muting
 	MutedChannels map[uint32]bool
+	userChannels  map[uint32]*gumble.Channel
 
 	// Added for noise suppression
 	NoiseSuppressor *noise.Suppressor
diff --git a/client.go b/client.go
index 7c11d55..d3a4095 100644
--- a/client.go
+++ b/client.go
@@ -78,6 +78,7 @@ func (b *Barnard) OnConnect(e *gumble.ConnectEvent) {
 
 	// Reset muted channels state on connect
 	b.MutedChannels = make(map[uint32]bool)
+	b.userChannels = make(map[uint32]*gumble.Channel)
 	b.RecordingMutex.Lock()
 	b.recordingAllowed = nil
 	b.recordingStarting = false
@@ -89,6 +90,7 @@ func (b *Barnard) OnConnect(e *gumble.ConnectEvent) {
 
 	for _, u := range b.Client.Users {
 		b.UserConfig.UpdateUser(u)
+		b.rememberUserChannel(u)
 	}
 
 	b.UpdateInputStatus(fmt.Sprintf("[%s]", e.Client.Self.Channel.Name))
@@ -169,6 +171,7 @@ func (b *Barnard) OnTextMessage(e *gumble.TextMessageEvent) {
 }
 
 func (b *Barnard) OnUserChange(e *gumble.UserChangeEvent) {
+	notification, hasNotification := b.userChangeNotification(e)
 	if e.User != nil {
 		b.UserConfig.UpdateUser(e.User)
 
@@ -187,28 +190,14 @@ func (b *Barnard) OnUserChange(e *gumble.UserChangeEvent) {
 		}
 	}
 
-	var s = "unknown"
-	var t = "unknown"
-	if e.Type.Has(gumble.UserChangeConnected) {
-		s = "joined"
-		t = "join"
-		// Notify about users joining our channel
-		if e.User.Channel.Name == b.Client.Self.Channel.Name {
-			b.Notify(t, e.User.Name, e.User.Channel.Name)
-			b.AddOutputLine(fmt.Sprintf("%s %s %s", e.User.Name, s, e.User.Channel.Name))
-		}
-	}
 	if e.Type.Has(gumble.UserChangeDisconnected) {
-		s = "left"
-		t = "leave"
 		if e.User == b.selectedUser {
 			b.SetSelectedUser(nil)
 		}
-		// Always notify about disconnects if user has channel info and was in our channel
-		if e.User.Channel != nil && e.User.Channel.Name == b.Client.Self.Channel.Name {
-			b.Notify(t, e.User.Name, e.User.Channel.Name)
-			b.AddOutputLine(fmt.Sprintf("%s %s %s", e.User.Name, s, e.User.Channel.Name))
-		}
+	}
+	if hasNotification {
+		b.Notify(notification.event, notification.who, notification.what)
+		b.AddOutputLine(notification.line)
 	}
 	if e.Type.Has(gumble.UserChangeChannel) && e.User == b.Client.Self {
 		b.UpdateInputStatus(fmt.Sprintf("[%s]", e.User.Channel.Name))
@@ -226,10 +215,94 @@ func (b *Barnard) OnUserChange(e *gumble.UserChangeEvent) {
 	if e.Type.Has(gumble.UserChangeStats) && e.User.Stats != nil {
 		b.AddOutputLine(formatUserStats(e.User))
 	}
+	b.updateUserChannel(e)
 	b.UiTree.Rebuild()
 	b.Ui.Refresh()
 }
 
+type userChangeNotification struct {
+	event string
+	who   string
+	what  string
+	line  string
+}
+
+func (b *Barnard) userChangeNotification(e *gumble.UserChangeEvent) (userChangeNotification, bool) {
+	if e == nil || e.User == nil || b.Client == nil || b.Client.Self == nil || b.Client.Self.Channel == nil {
+		return userChangeNotification{}, false
+	}
+
+	currentChannel := b.Client.Self.Channel
+	previousChannel := b.previousUserChannel(e.User)
+	userChannel := e.User.Channel
+
+	if e.Type.Has(gumble.UserChangeConnected) {
+		return buildUserChangeNotification("join", "joined", e.User, userChannel, currentChannel)
+	}
+	if e.Type.Has(gumble.UserChangeDisconnected) {
+		if previousChannel == nil {
+			previousChannel = userChannel
+		}
+		return buildUserChangeNotification("leave", "left", e.User, previousChannel, currentChannel)
+	}
+	if e.Type.Has(gumble.UserChangeChannel) && e.User != b.Client.Self {
+		if sameChannel(userChannel, currentChannel) && !sameChannel(previousChannel, currentChannel) {
+			return buildUserChangeNotification("join", "joined", e.User, userChannel, currentChannel)
+		}
+		if sameChannel(previousChannel, currentChannel) && !sameChannel(userChannel, currentChannel) {
+			return buildUserChangeNotification("leave", "left", e.User, previousChannel, currentChannel)
+		}
+	}
+
+	return userChangeNotification{}, false
+}
+
+func buildUserChangeNotification(event string, verb string, user *gumble.User, eventChannel *gumble.Channel, currentChannel *gumble.Channel) (userChangeNotification, bool) {
+	if !sameChannel(eventChannel, currentChannel) {
+		return userChangeNotification{}, false
+	}
+	return userChangeNotification{
+		event: event,
+		who:   user.Name,
+		what:  eventChannel.Name,
+		line:  fmt.Sprintf("%s %s %s", user.Name, verb, eventChannel.Name),
+	}, true
+}
+
+func sameChannel(a *gumble.Channel, b *gumble.Channel) bool {
+	return a != nil && b != nil && a.ID == b.ID
+}
+
+func (b *Barnard) previousUserChannel(user *gumble.User) *gumble.Channel {
+	if b.userChannels == nil || user == nil {
+		return nil
+	}
+	return b.userChannels[user.Session]
+}
+
+func (b *Barnard) rememberUserChannel(user *gumble.User) {
+	if user == nil || user.Channel == nil {
+		return
+	}
+	if b.userChannels == nil {
+		b.userChannels = make(map[uint32]*gumble.Channel)
+	}
+	b.userChannels[user.Session] = user.Channel
+}
+
+func (b *Barnard) updateUserChannel(e *gumble.UserChangeEvent) {
+	if e == nil || e.User == nil {
+		return
+	}
+	if e.Type.Has(gumble.UserChangeDisconnected) {
+		if b.userChannels != nil {
+			delete(b.userChannels, e.User.Session)
+		}
+		return
+	}
+	b.rememberUserChannel(e.User)
+}
+
 func (b *Barnard) OnChannelChange(e *gumble.ChannelChangeEvent) {
 	b.UpdateInputStatus(fmt.Sprintf("[%s]", e.Channel.Name))
 	if e.Type.Has(gumble.ChannelChangeDescription) {
diff --git a/client_notification_test.go b/client_notification_test.go
new file mode 100644
index 0000000..9eb0f74
--- /dev/null
+++ b/client_notification_test.go
@@ -0,0 +1,146 @@
+package main
+
+import (
+	"testing"
+
+	"git.stormux.org/storm/barnard/gumble/gumble"
+)
+
+func TestUserChangeNotification(t *testing.T) {
+	current := &gumble.Channel{ID: 1, Name: "Current"}
+	other := &gumble.Channel{ID: 2, Name: "Other"}
+	self := &gumble.User{Session: 1, Name: "Username", Channel: current}
+
+	tests := []struct {
+		name     string
+		user     *gumble.User
+		previous *gumble.Channel
+		change   gumble.UserChangeType
+		want     userChangeNotification
+		wantOK   bool
+	}{
+		{
+			name:   "connected to current channel",
+			user:   &gumble.User{Session: 2, Name: "Guest", Channel: current},
+			change: gumble.UserChangeConnected,
+			want: userChangeNotification{
+				event: "join",
+				who:   "Guest",
+				what:  "Current",
+				line:  "Guest joined Current",
+			},
+			wantOK: true,
+		},
+		{
+			name:     "disconnected from current channel",
+			user:     &gumble.User{Session: 2, Name: "Guest", Channel: current},
+			previous: current,
+			change:   gumble.UserChangeDisconnected,
+			want: userChangeNotification{
+				event: "leave",
+				who:   "Guest",
+				what:  "Current",
+				line:  "Guest left Current",
+			},
+			wantOK: true,
+		},
+		{
+			name:     "moved into current channel",
+			user:     &gumble.User{Session: 2, Name: "Guest", Channel: current},
+			previous: other,
+			change:   gumble.UserChangeChannel,
+			want: userChangeNotification{
+				event: "join",
+				who:   "Guest",
+				what:  "Current",
+				line:  "Guest joined Current",
+			},
+			wantOK: true,
+		},
+		{
+			name:     "moved out of current channel",
+			user:     &gumble.User{Session: 2, Name: "Guest", Channel: other},
+			previous: current,
+			change:   gumble.UserChangeChannel,
+			want: userChangeNotification{
+				event: "leave",
+				who:   "Guest",
+				what:  "Current",
+				line:  "Guest left Current",
+			},
+			wantOK: true,
+		},
+		{
+			name:   "connected to other channel",
+			user:   &gumble.User{Session: 2, Name: "Guest", Channel: other},
+			change: gumble.UserChangeConnected,
+		},
+		{
+			name:     "moved between other channels",
+			user:     &gumble.User{Session: 2, Name: "Guest", Channel: other},
+			previous: &gumble.Channel{ID: 3, Name: "Elsewhere"},
+			change:   gumble.UserChangeChannel,
+		},
+		{
+			name:     "self channel move",
+			user:     self,
+			previous: other,
+			change:   gumble.UserChangeChannel,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b := &Barnard{
+				Client: &gumble.Client{Self: self},
+			}
+			if tt.previous != nil {
+				b.userChannels = map[uint32]*gumble.Channel{tt.user.Session: tt.previous}
+			}
+
+			got, ok := b.userChangeNotification(&gumble.UserChangeEvent{
+				Client: b.Client,
+				Type:   tt.change,
+				User:   tt.user,
+			})
+			if ok != tt.wantOK {
+				t.Fatalf("expected ok %v, got %v", tt.wantOK, ok)
+			}
+			if got != tt.want {
+				t.Fatalf("expected %#v, got %#v", tt.want, got)
+			}
+		})
+	}
+}
+
+func TestUpdateUserChannel(t *testing.T) {
+	current := &gumble.Channel{ID: 1, Name: "Current"}
+	other := &gumble.Channel{ID: 2, Name: "Other"}
+	user := &gumble.User{Session: 2, Name: "Guest", Channel: current}
+	b := &Barnard{}
+
+	b.updateUserChannel(&gumble.UserChangeEvent{
+		Type: gumble.UserChangeConnected,
+		User: user,
+	})
+	if got := b.previousUserChannel(user); got != current {
+		t.Fatalf("expected current channel to be remembered, got %#v", got)
+	}
+
+	user.Channel = other
+	b.updateUserChannel(&gumble.UserChangeEvent{
+		Type: gumble.UserChangeChannel,
+		User: user,
+	})
+	if got := b.previousUserChannel(user); got != other {
+		t.Fatalf("expected other channel to be remembered, got %#v", got)
+	}
+
+	b.updateUserChannel(&gumble.UserChangeEvent{
+		Type: gumble.UserChangeDisconnected,
+		User: user,
+	})
+	if got := b.previousUserChannel(user); got != nil {
+		t.Fatalf("expected disconnected user channel to be removed, got %#v", got)
+	}
+}