diff --git a/client.go b/client.go
index 5f3c6ce..400bcce 100644
--- a/client.go
+++ b/client.go
@@ -702,13 +702,16 @@ func addDownTrack(c *client, conn *downConnection, remoteTrack *upTrack, remoteC
 		rate:           estimator.New(time.Second),
 	}
 	conn.tracks = append(conn.tracks, track)
-	remoteTrack.addLocal(track)
 
 	go rtcpDownListener(conn, track, s)
 
 	return s, nil
 }
 
+func activateDownTrack(conn *downConnection, track *downTrack) {
+	track.remote.addLocal(track)
+}
+
 const (
 	minLossRate  = 9600
 	initLossRate = 512 * 1000
@@ -1003,6 +1006,10 @@ func gotAnswer(c *client, id string, answer webrtc.SessionDescription) error {
 	if err != nil {
 		return err
 	}
+
+	for _, t := range conn.tracks {
+		activateDownTrack(conn, t)
+	}
 	return nil
 }