From 52f1c2b7952cb91cbbb51d0dab3a6a336f40faa4 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 29 Apr 2025 11:36:18 +0000 Subject: [PATCH 01/10] initial implementation --- tailnet/controllers.go | 10 +++- tailnet/controllers_test.go | 10 +++- vpn/tunnel.go | 57 ++++++++++++++++----- vpn/tunnel_internal_test.go | 98 +++++++++++++++++++++++++++++++++++++ 4 files changed, 159 insertions(+), 16 deletions(-) diff --git a/tailnet/controllers.go b/tailnet/controllers.go index 2328e19640a4d..34d28731f521c 100644 --- a/tailnet/controllers.go +++ b/tailnet/controllers.go @@ -1049,6 +1049,7 @@ func (t *tunnelUpdater) recvLoop() { t.logger.Debug(context.Background(), "tunnel updater recvLoop started") defer t.logger.Debug(context.Background(), "tunnel updater recvLoop done") defer close(t.recvLoopDone) + freshState := true for { update, err := t.client.Recv() if err != nil { @@ -1061,8 +1062,10 @@ func (t *tunnelUpdater) recvLoop() { } t.logger.Debug(context.Background(), "got workspace update", slog.F("workspace_update", update), + slog.F("fresh_state", freshState), ) - err = t.handleUpdate(update) + err = t.handleUpdate(update, freshState) + freshState = false if err != nil { t.logger.Critical(context.Background(), "failed to handle workspace Update", slog.Error(err)) cErr := t.client.Close() @@ -1083,6 +1086,7 @@ type WorkspaceUpdate struct { UpsertedAgents []*Agent DeletedWorkspaces []*Workspace DeletedAgents []*Agent + FreshState bool } func (w *WorkspaceUpdate) Clone() WorkspaceUpdate { @@ -1091,6 +1095,7 @@ func (w *WorkspaceUpdate) Clone() WorkspaceUpdate { UpsertedAgents: make([]*Agent, len(w.UpsertedAgents)), DeletedWorkspaces: make([]*Workspace, len(w.DeletedWorkspaces)), DeletedAgents: make([]*Agent, len(w.DeletedAgents)), + FreshState: w.FreshState, } for i, ws := range w.UpsertedWorkspaces { clone.UpsertedWorkspaces[i] = &Workspace{ @@ -1115,7 +1120,7 @@ func (w *WorkspaceUpdate) Clone() WorkspaceUpdate { return clone } -func (t *tunnelUpdater) handleUpdate(update *proto.WorkspaceUpdate) error { +func (t *tunnelUpdater) handleUpdate(update *proto.WorkspaceUpdate, freshState bool) error { t.Lock() defer t.Unlock() @@ -1124,6 +1129,7 @@ func (t *tunnelUpdater) handleUpdate(update *proto.WorkspaceUpdate) error { UpsertedAgents: []*Agent{}, DeletedWorkspaces: []*Workspace{}, DeletedAgents: []*Agent{}, + FreshState: freshState, } for _, uw := range update.UpsertedWorkspaces { diff --git a/tailnet/controllers_test.go b/tailnet/controllers_test.go index 67834de462655..eac6a15326007 100644 --- a/tailnet/controllers_test.go +++ b/tailnet/controllers_test.go @@ -1611,13 +1611,15 @@ func TestTunnelAllWorkspaceUpdatesController_Initial(t *testing.T) { }, DeletedWorkspaces: []*tailnet.Workspace{}, DeletedAgents: []*tailnet.Agent{}, + FreshState: true, } // And the callback cbUpdate := testutil.TryReceive(ctx, t, fUH.ch) require.Equal(t, currentState, cbUpdate) - // Current recvState should match + // Current recvState should match but shouldn't be a fresh state + currentState.FreshState = false recvState, err := updateCtrl.CurrentState() require.NoError(t, err) slices.SortFunc(recvState.UpsertedWorkspaces, func(a, b *tailnet.Workspace) int { @@ -1692,12 +1694,14 @@ func TestTunnelAllWorkspaceUpdatesController_DeleteAgent(t *testing.T) { }, DeletedWorkspaces: []*tailnet.Workspace{}, DeletedAgents: []*tailnet.Agent{}, + FreshState: true, } cbUpdate := testutil.TryReceive(ctx, t, fUH.ch) require.Equal(t, initRecvUp, cbUpdate) - // Current state should match initial + // Current state should match initial but shouldn't be a fresh state + initRecvUp.FreshState = false state, err := updateCtrl.CurrentState() require.NoError(t, err) require.Equal(t, initRecvUp, state) @@ -1753,6 +1757,7 @@ func TestTunnelAllWorkspaceUpdatesController_DeleteAgent(t *testing.T) { "w1.coder.": {ws1a1IP}, }}, }, + FreshState: false, } require.Equal(t, sndRecvUpdate, cbUpdate) @@ -1771,6 +1776,7 @@ func TestTunnelAllWorkspaceUpdatesController_DeleteAgent(t *testing.T) { }, DeletedWorkspaces: []*tailnet.Workspace{}, DeletedAgents: []*tailnet.Agent{}, + FreshState: false, }, state) } diff --git a/vpn/tunnel.go b/vpn/tunnel.go index 63de203980d14..3a561cb67b62f 100644 --- a/vpn/tunnel.go +++ b/vpn/tunnel.go @@ -397,6 +397,42 @@ func (u *updater) sendUpdateResponse(req *request[*TunnelMessage, *ManagerMessag // createPeerUpdateLocked creates a PeerUpdate message from a workspace update, populating // the network status of the agents. func (u *updater) createPeerUpdateLocked(update tailnet.WorkspaceUpdate) *PeerUpdate { + // this flag is true on the first update after a reconnect + if update.FreshState { + // ignoredWorkspaces is initially populated with the workspaces that are + // in the current update. Later on we populate it with the deleted workspaces too + // so that we don't send duplicate updates. Same applies to ignoredAgents. + ignoredWorkspaces := make(map[uuid.UUID]struct{}, len(update.UpsertedWorkspaces)) + ignoredAgents := make(map[uuid.UUID]struct{}, len(update.UpsertedAgents)) + + for _, workspace := range update.UpsertedWorkspaces { + ignoredWorkspaces[workspace.ID] = struct{}{} + } + for _, agent := range update.UpsertedAgents { + ignoredAgents[agent.ID] = struct{}{} + } + for _, agent := range u.agents { + if _, ok := ignoredAgents[agent.ID]; !ok { + // delete any current agents that are not in the new update + update.DeletedAgents = append(update.DeletedAgents, &tailnet.Agent{ + ID: agent.ID, + Name: agent.Name, + WorkspaceID: agent.WorkspaceID, + }) + // if the workspace connected to an agent we're deleting, + // is not present in the fresh state, add it to the deleted workspaces + if _, ok := ignoredWorkspaces[agent.WorkspaceID]; !ok { + update.DeletedWorkspaces = append(update.DeletedWorkspaces, &tailnet.Workspace{ + // other fields cannot be populated because the tunnel + // only stores agents and corresponding workspaceIDs + ID: agent.WorkspaceID, + }) + ignoredWorkspaces[agent.WorkspaceID] = struct{}{} + } + } + } + } + out := &PeerUpdate{ UpsertedWorkspaces: make([]*Workspace, len(update.UpsertedWorkspaces)), UpsertedAgents: make([]*Agent, len(update.UpsertedAgents)), @@ -404,7 +440,14 @@ func (u *updater) createPeerUpdateLocked(update tailnet.WorkspaceUpdate) *PeerUp DeletedAgents: make([]*Agent, len(update.DeletedAgents)), } - u.saveUpdateLocked(update) + // save the workspace update to the tunnel's state, such that it can + // be used to populate automated peer updates. + for _, agent := range update.UpsertedAgents { + u.agents[agent.ID] = agent.Clone() + } + for _, agent := range update.DeletedAgents { + delete(u.agents, agent.ID) + } for i, ws := range update.UpsertedWorkspaces { out.UpsertedWorkspaces[i] = &Workspace{ @@ -413,6 +456,7 @@ func (u *updater) createPeerUpdateLocked(update tailnet.WorkspaceUpdate) *PeerUp Status: Workspace_Status(ws.Status), } } + upsertedAgents := u.convertAgentsLocked(update.UpsertedAgents) out.UpsertedAgents = upsertedAgents for i, ws := range update.DeletedWorkspaces { @@ -472,17 +516,6 @@ func (u *updater) convertAgentsLocked(agents []*tailnet.Agent) []*Agent { return out } -// saveUpdateLocked saves the workspace update to the tunnel's state, such that it can -// be used to populate automated peer updates. -func (u *updater) saveUpdateLocked(update tailnet.WorkspaceUpdate) { - for _, agent := range update.UpsertedAgents { - u.agents[agent.ID] = agent.Clone() - } - for _, agent := range update.DeletedAgents { - delete(u.agents, agent.ID) - } -} - // setConn sets the `conn` and returns false if there's already a connection set. func (u *updater) setConn(conn Conn) bool { u.mu.Lock() diff --git a/vpn/tunnel_internal_test.go b/vpn/tunnel_internal_test.go index d1d7377361f79..d7211b88a065c 100644 --- a/vpn/tunnel_internal_test.go +++ b/vpn/tunnel_internal_test.go @@ -486,6 +486,104 @@ func TestTunnel_sendAgentUpdate(t *testing.T) { require.Equal(t, hsTime, req.msg.GetPeerUpdate().UpsertedAgents[0].LastHandshake.AsTime()) } +func TestTunnel_sendAgentUpdateReconnect(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + + mClock := quartz.NewMock(t) + + wID1 := uuid.UUID{1} + aID1 := uuid.UUID{2} + aID2 := uuid.UUID{3} + hsTime := time.Now().Add(-time.Minute).UTC() + + client := newFakeClient(ctx, t) + conn := newFakeConn(tailnet.WorkspaceUpdate{}, hsTime) + + tun, mgr := setupTunnel(t, ctx, client, mClock) + errCh := make(chan error, 1) + var resp *TunnelMessage + go func() { + r, err := mgr.unaryRPC(ctx, &ManagerMessage{ + Msg: &ManagerMessage_Start{ + Start: &StartRequest{ + TunnelFileDescriptor: 2, + CoderUrl: "https://coder.example.com", + ApiToken: "fakeToken", + }, + }, + }) + resp = r + errCh <- err + }() + testutil.RequireSend(ctx, t, client.ch, conn) + err := testutil.TryReceive(ctx, t, errCh) + require.NoError(t, err) + _, ok := resp.Msg.(*TunnelMessage_Start) + require.True(t, ok) + + // Inform the tunnel of the initial state + err = tun.Update(tailnet.WorkspaceUpdate{ + UpsertedWorkspaces: []*tailnet.Workspace{ + { + ID: wID1, Name: "w1", Status: proto.Workspace_STARTING, + }, + }, + UpsertedAgents: []*tailnet.Agent{ + { + ID: aID1, + Name: "agent1", + WorkspaceID: wID1, + Hosts: map[dnsname.FQDN][]netip.Addr{ + "agent1.coder.": {netip.MustParseAddr("fd60:627a:a42b:0101::")}, + }, + }, + }, + }) + require.NoError(t, err) + req := testutil.TryReceive(ctx, t, mgr.requests) + require.Nil(t, req.msg.Rpc) + require.NotNil(t, req.msg.GetPeerUpdate()) + require.Len(t, req.msg.GetPeerUpdate().UpsertedAgents, 1) + require.Equal(t, aID1[:], req.msg.GetPeerUpdate().UpsertedAgents[0].Id) + + // Upsert a new agent simulating a reconnect + err = tun.Update(tailnet.WorkspaceUpdate{ + UpsertedWorkspaces: []*tailnet.Workspace{ + { + ID: wID1, Name: "w1", Status: proto.Workspace_STARTING, + }, + }, + UpsertedAgents: []*tailnet.Agent{ + { + ID: aID2, + Name: "agent2", + WorkspaceID: wID1, + Hosts: map[dnsname.FQDN][]netip.Addr{ + "agent2.coder.": {netip.MustParseAddr("fd60:627a:a42b:0101::")}, + }, + }, + }, + FreshState: true, + }) + require.NoError(t, err) + + // The new update only contains the new agent + mClock.AdvanceNext() + req = testutil.TryReceive(ctx, t, mgr.requests) + require.Nil(t, req.msg.Rpc) + peerUpdate := req.msg.GetPeerUpdate() + require.NotNil(t, peerUpdate) + require.Len(t, peerUpdate.UpsertedAgents, 1) + require.Len(t, peerUpdate.DeletedAgents, 1) + + require.Equal(t, aID2[:], peerUpdate.UpsertedAgents[0].Id) + require.Equal(t, hsTime, peerUpdate.UpsertedAgents[0].LastHandshake.AsTime()) + + require.Equal(t, aID1[:], peerUpdate.DeletedAgents[0].Id) +} + //nolint:revive // t takes precedence func setupTunnel(t *testing.T, ctx context.Context, client *fakeClient, mClock quartz.Clock) (*Tunnel, *speaker[*ManagerMessage, *TunnelMessage, TunnelMessage]) { mp, tp := net.Pipe() From 288c33e0aa96b615185bfe348fe9fd948c705f07 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 29 Apr 2025 12:29:01 +0000 Subject: [PATCH 02/10] moved out the logic to separate function --- vpn/tunnel.go | 69 +++++++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/vpn/tunnel.go b/vpn/tunnel.go index 3a561cb67b62f..7640a07165269 100644 --- a/vpn/tunnel.go +++ b/vpn/tunnel.go @@ -394,43 +394,48 @@ func (u *updater) sendUpdateResponse(req *request[*TunnelMessage, *ManagerMessag return nil } +// processFreshState handles the logic for when a fresh state update is received. +func (u *updater) processFreshState(update *tailnet.WorkspaceUpdate) { + // ignoredWorkspaces is initially populated with the workspaces that are + // in the current update. Later on we populate it with the deleted workspaces too + // so that we don't send duplicate updates. Same applies to ignoredAgents. + ignoredWorkspaces := make(map[uuid.UUID]struct{}, len(update.UpsertedWorkspaces)) + ignoredAgents := make(map[uuid.UUID]struct{}, len(update.UpsertedAgents)) + + for _, workspace := range update.UpsertedWorkspaces { + ignoredWorkspaces[workspace.ID] = struct{}{} + } + for _, agent := range update.UpsertedAgents { + ignoredAgents[agent.ID] = struct{}{} + } + for _, agent := range u.agents { + if _, ok := ignoredAgents[agent.ID]; !ok { + // delete any current agents that are not in the new update + update.DeletedAgents = append(update.DeletedAgents, &tailnet.Agent{ + ID: agent.ID, + Name: agent.Name, + WorkspaceID: agent.WorkspaceID, + }) + // if the workspace connected to an agent we're deleting, + // is not present in the fresh state, add it to the deleted workspaces + if _, ok := ignoredWorkspaces[agent.WorkspaceID]; !ok { + update.DeletedWorkspaces = append(update.DeletedWorkspaces, &tailnet.Workspace{ + // other fields cannot be populated because the tunnel + // only stores agents and corresponding workspaceIDs + ID: agent.WorkspaceID, + }) + ignoredWorkspaces[agent.WorkspaceID] = struct{}{} + } + } + } +} + // createPeerUpdateLocked creates a PeerUpdate message from a workspace update, populating // the network status of the agents. func (u *updater) createPeerUpdateLocked(update tailnet.WorkspaceUpdate) *PeerUpdate { // this flag is true on the first update after a reconnect if update.FreshState { - // ignoredWorkspaces is initially populated with the workspaces that are - // in the current update. Later on we populate it with the deleted workspaces too - // so that we don't send duplicate updates. Same applies to ignoredAgents. - ignoredWorkspaces := make(map[uuid.UUID]struct{}, len(update.UpsertedWorkspaces)) - ignoredAgents := make(map[uuid.UUID]struct{}, len(update.UpsertedAgents)) - - for _, workspace := range update.UpsertedWorkspaces { - ignoredWorkspaces[workspace.ID] = struct{}{} - } - for _, agent := range update.UpsertedAgents { - ignoredAgents[agent.ID] = struct{}{} - } - for _, agent := range u.agents { - if _, ok := ignoredAgents[agent.ID]; !ok { - // delete any current agents that are not in the new update - update.DeletedAgents = append(update.DeletedAgents, &tailnet.Agent{ - ID: agent.ID, - Name: agent.Name, - WorkspaceID: agent.WorkspaceID, - }) - // if the workspace connected to an agent we're deleting, - // is not present in the fresh state, add it to the deleted workspaces - if _, ok := ignoredWorkspaces[agent.WorkspaceID]; !ok { - update.DeletedWorkspaces = append(update.DeletedWorkspaces, &tailnet.Workspace{ - // other fields cannot be populated because the tunnel - // only stores agents and corresponding workspaceIDs - ID: agent.WorkspaceID, - }) - ignoredWorkspaces[agent.WorkspaceID] = struct{}{} - } - } - } + u.processFreshState(&update) } out := &PeerUpdate{ From 0de5df22a11816b4aff791d297ff6d8a739b04da Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:19:25 +0000 Subject: [PATCH 03/10] additional tests --- vpn/tunnel.go | 74 ++++----- vpn/tunnel_internal_test.go | 304 ++++++++++++++++++++++++++++++++++++ 2 files changed, 341 insertions(+), 37 deletions(-) diff --git a/vpn/tunnel.go b/vpn/tunnel.go index 7640a07165269..4731ad62606d5 100644 --- a/vpn/tunnel.go +++ b/vpn/tunnel.go @@ -394,48 +394,12 @@ func (u *updater) sendUpdateResponse(req *request[*TunnelMessage, *ManagerMessag return nil } -// processFreshState handles the logic for when a fresh state update is received. -func (u *updater) processFreshState(update *tailnet.WorkspaceUpdate) { - // ignoredWorkspaces is initially populated with the workspaces that are - // in the current update. Later on we populate it with the deleted workspaces too - // so that we don't send duplicate updates. Same applies to ignoredAgents. - ignoredWorkspaces := make(map[uuid.UUID]struct{}, len(update.UpsertedWorkspaces)) - ignoredAgents := make(map[uuid.UUID]struct{}, len(update.UpsertedAgents)) - - for _, workspace := range update.UpsertedWorkspaces { - ignoredWorkspaces[workspace.ID] = struct{}{} - } - for _, agent := range update.UpsertedAgents { - ignoredAgents[agent.ID] = struct{}{} - } - for _, agent := range u.agents { - if _, ok := ignoredAgents[agent.ID]; !ok { - // delete any current agents that are not in the new update - update.DeletedAgents = append(update.DeletedAgents, &tailnet.Agent{ - ID: agent.ID, - Name: agent.Name, - WorkspaceID: agent.WorkspaceID, - }) - // if the workspace connected to an agent we're deleting, - // is not present in the fresh state, add it to the deleted workspaces - if _, ok := ignoredWorkspaces[agent.WorkspaceID]; !ok { - update.DeletedWorkspaces = append(update.DeletedWorkspaces, &tailnet.Workspace{ - // other fields cannot be populated because the tunnel - // only stores agents and corresponding workspaceIDs - ID: agent.WorkspaceID, - }) - ignoredWorkspaces[agent.WorkspaceID] = struct{}{} - } - } - } -} - // createPeerUpdateLocked creates a PeerUpdate message from a workspace update, populating // the network status of the agents. func (u *updater) createPeerUpdateLocked(update tailnet.WorkspaceUpdate) *PeerUpdate { // this flag is true on the first update after a reconnect if update.FreshState { - u.processFreshState(&update) + processFreshState(&update, u.agents) } out := &PeerUpdate{ @@ -590,6 +554,42 @@ func (u *updater) netStatusLoop() { } } +// processFreshState handles the logic for when a fresh state update is received. +func processFreshState(update *tailnet.WorkspaceUpdate, agents map[uuid.UUID]tailnet.Agent) { + // ignoredWorkspaces is initially populated with the workspaces that are + // in the current update. Later on we populate it with the deleted workspaces too + // so that we don't send duplicate updates. Same applies to ignoredAgents. + ignoredWorkspaces := make(map[uuid.UUID]struct{}, len(update.UpsertedWorkspaces)) + ignoredAgents := make(map[uuid.UUID]struct{}, len(update.UpsertedAgents)) + + for _, workspace := range update.UpsertedWorkspaces { + ignoredWorkspaces[workspace.ID] = struct{}{} + } + for _, agent := range update.UpsertedAgents { + ignoredAgents[agent.ID] = struct{}{} + } + for _, agent := range agents { + if _, ok := ignoredAgents[agent.ID]; !ok { + // delete any current agents that are not in the new update + update.DeletedAgents = append(update.DeletedAgents, &tailnet.Agent{ + ID: agent.ID, + Name: agent.Name, + WorkspaceID: agent.WorkspaceID, + }) + // if the workspace connected to an agent we're deleting, + // is not present in the fresh state, add it to the deleted workspaces + if _, ok := ignoredWorkspaces[agent.WorkspaceID]; !ok { + update.DeletedWorkspaces = append(update.DeletedWorkspaces, &tailnet.Workspace{ + // other fields cannot be populated because the tunnel + // only stores agents and corresponding workspaceIDs + ID: agent.WorkspaceID, + }) + ignoredWorkspaces[agent.WorkspaceID] = struct{}{} + } + } + } +} + // hostsToIPStrings returns a slice of all unique IP addresses in the values // of the given map. func hostsToIPStrings(hosts map[dnsname.FQDN][]netip.Addr) []string { diff --git a/vpn/tunnel_internal_test.go b/vpn/tunnel_internal_test.go index d7211b88a065c..b12b4b5073b3f 100644 --- a/vpn/tunnel_internal_test.go +++ b/vpn/tunnel_internal_test.go @@ -6,6 +6,7 @@ import ( "net/netip" "net/url" "slices" + "sort" "strings" "sync" "testing" @@ -496,6 +497,7 @@ func TestTunnel_sendAgentUpdateReconnect(t *testing.T) { wID1 := uuid.UUID{1} aID1 := uuid.UUID{2} aID2 := uuid.UUID{3} + hsTime := time.Now().Add(-time.Minute).UTC() client := newFakeClient(ctx, t) @@ -584,6 +586,112 @@ func TestTunnel_sendAgentUpdateReconnect(t *testing.T) { require.Equal(t, aID1[:], peerUpdate.DeletedAgents[0].Id) } +func TestTunnel_sendAgentUpdateWorkspaceReconnect(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + + mClock := quartz.NewMock(t) + + wID1 := uuid.UUID{1} + wID2 := uuid.UUID{2} + aID1 := uuid.UUID{3} + aID3 := uuid.UUID{4} + + hsTime := time.Now().Add(-time.Minute).UTC() + + client := newFakeClient(ctx, t) + conn := newFakeConn(tailnet.WorkspaceUpdate{}, hsTime) + + tun, mgr := setupTunnel(t, ctx, client, mClock) + errCh := make(chan error, 1) + var resp *TunnelMessage + go func() { + r, err := mgr.unaryRPC(ctx, &ManagerMessage{ + Msg: &ManagerMessage_Start{ + Start: &StartRequest{ + TunnelFileDescriptor: 2, + CoderUrl: "https://coder.example.com", + ApiToken: "fakeToken", + }, + }, + }) + resp = r + errCh <- err + }() + testutil.RequireSend(ctx, t, client.ch, conn) + err := testutil.TryReceive(ctx, t, errCh) + require.NoError(t, err) + _, ok := resp.Msg.(*TunnelMessage_Start) + require.True(t, ok) + + // Inform the tunnel of the initial state + err = tun.Update(tailnet.WorkspaceUpdate{ + UpsertedWorkspaces: []*tailnet.Workspace{ + { + ID: wID1, Name: "w1", Status: proto.Workspace_STARTING, + }, + }, + UpsertedAgents: []*tailnet.Agent{ + { + ID: aID1, + Name: "agent1", + WorkspaceID: wID1, + Hosts: map[dnsname.FQDN][]netip.Addr{ + "agent1.coder.": {netip.MustParseAddr("fd60:627a:a42b:0101::")}, + }, + }, + }, + }) + require.NoError(t, err) + req := testutil.TryReceive(ctx, t, mgr.requests) + require.Nil(t, req.msg.Rpc) + require.NotNil(t, req.msg.GetPeerUpdate()) + require.Len(t, req.msg.GetPeerUpdate().UpsertedAgents, 1) + require.Equal(t, aID1[:], req.msg.GetPeerUpdate().UpsertedAgents[0].Id) + + // Upsert a new agent with a new workspace while simulating a reconnect + err = tun.Update(tailnet.WorkspaceUpdate{ + UpsertedWorkspaces: []*tailnet.Workspace{ + { + ID: wID2, Name: "w2", Status: proto.Workspace_STARTING, + }, + }, + UpsertedAgents: []*tailnet.Agent{ + { + ID: aID3, + Name: "agent3", + WorkspaceID: wID2, + Hosts: map[dnsname.FQDN][]netip.Addr{ + "agent3.coder.": {netip.MustParseAddr("fd60:627a:a42b:0101::")}, + }, + }, + }, + FreshState: true, + }) + require.NoError(t, err) + + // The new update only contains the new agent + mClock.AdvanceNext() + req = testutil.TryReceive(ctx, t, mgr.requests) + mClock.AdvanceNext() + + require.Nil(t, req.msg.Rpc) + peerUpdate := req.msg.GetPeerUpdate() + require.NotNil(t, peerUpdate) + require.Len(t, peerUpdate.UpsertedWorkspaces, 1) + require.Len(t, peerUpdate.UpsertedAgents, 1) + require.Len(t, peerUpdate.DeletedAgents, 1) + require.Len(t, peerUpdate.DeletedWorkspaces, 1) + + require.Equal(t, wID2[:], peerUpdate.UpsertedWorkspaces[0].Id) + require.Equal(t, aID3[:], peerUpdate.UpsertedAgents[0].Id) + require.Equal(t, hsTime, peerUpdate.UpsertedAgents[0].LastHandshake.AsTime()) + + require.Equal(t, aID1[:], peerUpdate.DeletedAgents[0].Id) + require.Equal(t, wID1[:], peerUpdate.DeletedWorkspaces[0].Id) +} + //nolint:revive // t takes precedence func setupTunnel(t *testing.T, ctx context.Context, client *fakeClient, mClock quartz.Clock) (*Tunnel, *speaker[*ManagerMessage, *TunnelMessage, TunnelMessage]) { mp, tp := net.Pipe() @@ -611,3 +719,199 @@ func setupTunnel(t *testing.T, ctx context.Context, client *fakeClient, mClock q mgr.start() return tun, mgr } + +func TestProcessFreshState(t *testing.T) { + t.Parallel() + + wsID1 := uuid.New() + wsID2 := uuid.New() + wsID3 := uuid.New() + + agentID1 := uuid.New() + agentID2 := uuid.New() + agentID3 := uuid.New() + agentID4 := uuid.New() // Belongs to wsID1 + + agent1 := tailnet.Agent{ID: agentID1, Name: "agent1", WorkspaceID: wsID1} + agent2 := tailnet.Agent{ID: agentID2, Name: "agent2", WorkspaceID: wsID2} + agent3 := tailnet.Agent{ID: agentID3, Name: "agent3", WorkspaceID: wsID3} + agent4 := tailnet.Agent{ID: agentID4, Name: "agent4", WorkspaceID: wsID1} // Same workspace as agent1 + + ws1 := tailnet.Workspace{ID: wsID1, Name: "ws1"} + ws2 := tailnet.Workspace{ID: wsID2, Name: "ws2"} + ws3 := tailnet.Workspace{ID: wsID3, Name: "ws3"} + + initialAgents := map[uuid.UUID]tailnet.Agent{ + agentID1: agent1, + agentID2: agent2, + agentID4: agent4, // agent 4 is initially present + } + + tests := []struct { + name string + initialAgents map[uuid.UUID]tailnet.Agent + update *tailnet.WorkspaceUpdate + expectedDelete *tailnet.WorkspaceUpdate // We only care about deletions added by the function + }{ + { + name: "NoChange", + initialAgents: initialAgents, + update: &tailnet.WorkspaceUpdate{ + FreshState: true, + UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, + UpsertedAgents: []*tailnet.Agent{&agent1, &agent2, &agent4}, + DeletedWorkspaces: []*tailnet.Workspace{}, // Input deletions + DeletedAgents: []*tailnet.Agent{}, // Input deletions + }, + expectedDelete: &tailnet.WorkspaceUpdate{ // Expect no *additional* deletions + DeletedWorkspaces: []*tailnet.Workspace{}, + DeletedAgents: []*tailnet.Agent{}, + }, + }, + { + name: "AgentAdded", // Agent 3 added in update + initialAgents: initialAgents, + update: &tailnet.WorkspaceUpdate{ + FreshState: true, + UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2, &ws3}, + UpsertedAgents: []*tailnet.Agent{&agent1, &agent2, &agent3, &agent4}, + DeletedWorkspaces: []*tailnet.Workspace{}, + DeletedAgents: []*tailnet.Agent{}, + }, + expectedDelete: &tailnet.WorkspaceUpdate{ + DeletedWorkspaces: []*tailnet.Workspace{}, + DeletedAgents: []*tailnet.Agent{}, + }, + }, + { + name: "AgentRemovedWorkspaceAlsoRemoved", // Agent 2 removed, ws2 also removed + initialAgents: initialAgents, + update: &tailnet.WorkspaceUpdate{ + FreshState: true, + UpsertedWorkspaces: []*tailnet.Workspace{&ws1}, // ws2 not present + UpsertedAgents: []*tailnet.Agent{&agent1, &agent4}, // agent2 not present + DeletedWorkspaces: []*tailnet.Workspace{}, + DeletedAgents: []*tailnet.Agent{}, + }, + expectedDelete: &tailnet.WorkspaceUpdate{ + DeletedWorkspaces: []*tailnet.Workspace{{ID: wsID2}}, // Expect ws2 to be deleted + DeletedAgents: []*tailnet.Agent{ // Expect agent2 to be deleted + {ID: agentID2, Name: "agent2", WorkspaceID: wsID2}, + }, + }, + }, + { + name: "AgentRemovedWorkspaceStays", // Agent 4 removed, but ws1 stays (due to agent1) + initialAgents: initialAgents, + update: &tailnet.WorkspaceUpdate{ + FreshState: true, + UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, // ws1 still present + UpsertedAgents: []*tailnet.Agent{&agent1, &agent2}, // agent4 not present + DeletedWorkspaces: []*tailnet.Workspace{}, + DeletedAgents: []*tailnet.Agent{}, + }, + expectedDelete: &tailnet.WorkspaceUpdate{ + DeletedWorkspaces: []*tailnet.Workspace{}, // ws1 should NOT be deleted + DeletedAgents: []*tailnet.Agent{ // Expect agent4 to be deleted + {ID: agentID4, Name: "agent4", WorkspaceID: wsID1}, + }, + }, + }, + { + name: "InitialAgentsEmpty", + initialAgents: map[uuid.UUID]tailnet.Agent{}, // Start with no agents known + update: &tailnet.WorkspaceUpdate{ + FreshState: true, + UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, + UpsertedAgents: []*tailnet.Agent{&agent1, &agent2}, + DeletedWorkspaces: []*tailnet.Workspace{}, + DeletedAgents: []*tailnet.Agent{}, + }, + expectedDelete: &tailnet.WorkspaceUpdate{ // Expect no deletions added + DeletedWorkspaces: []*tailnet.Workspace{}, + DeletedAgents: []*tailnet.Agent{}, + }, + }, + { + name: "UpdateEmpty", // Fresh state says nothing exists + initialAgents: initialAgents, + update: &tailnet.WorkspaceUpdate{ + FreshState: true, + UpsertedWorkspaces: []*tailnet.Workspace{}, + UpsertedAgents: []*tailnet.Agent{}, + DeletedWorkspaces: []*tailnet.Workspace{}, + DeletedAgents: []*tailnet.Agent{}, + }, + expectedDelete: &tailnet.WorkspaceUpdate{ // Expect all initial agents/workspaces to be deleted + DeletedWorkspaces: []*tailnet.Workspace{{ID: wsID1}, {ID: wsID2}}, // ws1 and ws2 deleted + DeletedAgents: []*tailnet.Agent{ // agent1, agent2, agent4 deleted + {ID: agentID1, Name: "agent1", WorkspaceID: wsID1}, + {ID: agentID2, Name: "agent2", WorkspaceID: wsID2}, + {ID: agentID4, Name: "agent4", WorkspaceID: wsID1}, + }, + }, + }, + { + name: "UpdateWithExistingDeletions", // Agent 2 removed, ws2 also removed, but update already contains some deletions + initialAgents: initialAgents, + update: &tailnet.WorkspaceUpdate{ + FreshState: true, + UpsertedWorkspaces: []*tailnet.Workspace{&ws1}, // ws2 not present + UpsertedAgents: []*tailnet.Agent{&agent1, &agent4}, // agent2 not present + DeletedWorkspaces: []*tailnet.Workspace{{ID: uuid.New()}}, // Pre-existing deletion + DeletedAgents: []*tailnet.Agent{{ID: uuid.New()}}, // Pre-existing deletion + }, + expectedDelete: &tailnet.WorkspaceUpdate{ + DeletedWorkspaces: []*tailnet.Workspace{{ID: wsID2}}, // Expect ws2 to be added to deletions + DeletedAgents: []*tailnet.Agent{ // Expect agent2 to be added to deletions + {ID: agentID2, Name: "agent2", WorkspaceID: wsID2}, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // Keep track of original lengths to compare only the added elements + originalDeletedAgentsLen := len(tt.update.DeletedAgents) + originalDeletedWorkspacesLen := len(tt.update.DeletedWorkspaces) + + agentsCopy := make(map[uuid.UUID]tailnet.Agent) + for k, v := range tt.initialAgents { + agentsCopy[k] = v + } + + processFreshState(tt.update, agentsCopy) + + // Extract only the elements added by processFreshState + addedDeletedAgents := tt.update.DeletedAgents[originalDeletedAgentsLen:] + addedDeletedWorkspaces := tt.update.DeletedWorkspaces[originalDeletedWorkspacesLen:] + + // Sort slices for consistent comparison + sortAgents(addedDeletedAgents) + sortAgents(tt.expectedDelete.DeletedAgents) + sortWorkspaces(addedDeletedWorkspaces) + sortWorkspaces(tt.expectedDelete.DeletedWorkspaces) + + require.ElementsMatch(t, tt.expectedDelete.DeletedAgents, addedDeletedAgents, "DeletedAgents mismatch") + require.ElementsMatch(t, tt.expectedDelete.DeletedWorkspaces, addedDeletedWorkspaces, "DeletedWorkspaces mismatch") + }) + } +} + +// sortAgents sorts a slice of Agents by ID for consistent comparison. +func sortAgents(agents []*tailnet.Agent) { + sort.Slice(agents, func(i, j int) bool { + return agents[i].ID.String() < agents[j].ID.String() + }) +} + +// sortWorkspaces sorts a slice of Workspaces by ID for consistent comparison. +func sortWorkspaces(workspaces []*tailnet.Workspace) { + sort.Slice(workspaces, func(i, j int) bool { + return workspaces[i].ID.String() < workspaces[j].ID.String() + }) +} From 2b77b8e5f2de062cb77340f914731dba6a52fa0e Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:20:06 +0000 Subject: [PATCH 04/10] fixed map copy --- vpn/tunnel_internal_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vpn/tunnel_internal_test.go b/vpn/tunnel_internal_test.go index b12b4b5073b3f..1ff283634f10d 100644 --- a/vpn/tunnel_internal_test.go +++ b/vpn/tunnel_internal_test.go @@ -19,6 +19,8 @@ import ( "github.com/coder/quartz" + "maps" + "github.com/coder/coder/v2/tailnet" "github.com/coder/coder/v2/tailnet/proto" "github.com/coder/coder/v2/testutil" @@ -880,9 +882,7 @@ func TestProcessFreshState(t *testing.T) { originalDeletedWorkspacesLen := len(tt.update.DeletedWorkspaces) agentsCopy := make(map[uuid.UUID]tailnet.Agent) - for k, v := range tt.initialAgents { - agentsCopy[k] = v - } + maps.Copy(agentsCopy, tt.initialAgents) processFreshState(tt.update, agentsCopy) From 3d3971359763dfa8bd03a0ccee113b7e0caf2e39 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:35:04 +0000 Subject: [PATCH 05/10] fmt & lint fixes --- vpn/tunnel_internal_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vpn/tunnel_internal_test.go b/vpn/tunnel_internal_test.go index 1ff283634f10d..df414c3434dcd 100644 --- a/vpn/tunnel_internal_test.go +++ b/vpn/tunnel_internal_test.go @@ -2,6 +2,7 @@ package vpn import ( "context" + "maps" "net" "net/netip" "net/url" @@ -19,8 +20,6 @@ import ( "github.com/coder/quartz" - "maps" - "github.com/coder/coder/v2/tailnet" "github.com/coder/coder/v2/tailnet/proto" "github.com/coder/coder/v2/testutil" From ca2e1bf02a69396e79c1317af1043a118d53b70d Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 30 Apr 2025 12:40:25 +0000 Subject: [PATCH 06/10] removed unecessary test for verifying existing deletetions (fresh state never has them) --- vpn/tunnel_internal_test.go | 60 +++++-------------------------------- 1 file changed, 7 insertions(+), 53 deletions(-) diff --git a/vpn/tunnel_internal_test.go b/vpn/tunnel_internal_test.go index df414c3434dcd..0dca9ca0be59c 100644 --- a/vpn/tunnel_internal_test.go +++ b/vpn/tunnel_internal_test.go @@ -7,7 +7,6 @@ import ( "net/netip" "net/url" "slices" - "sort" "strings" "sync" "testing" @@ -731,12 +730,12 @@ func TestProcessFreshState(t *testing.T) { agentID1 := uuid.New() agentID2 := uuid.New() agentID3 := uuid.New() - agentID4 := uuid.New() // Belongs to wsID1 + agentID4 := uuid.New() agent1 := tailnet.Agent{ID: agentID1, Name: "agent1", WorkspaceID: wsID1} agent2 := tailnet.Agent{ID: agentID2, Name: "agent2", WorkspaceID: wsID2} agent3 := tailnet.Agent{ID: agentID3, Name: "agent3", WorkspaceID: wsID3} - agent4 := tailnet.Agent{ID: agentID4, Name: "agent4", WorkspaceID: wsID1} // Same workspace as agent1 + agent4 := tailnet.Agent{ID: agentID4, Name: "agent4", WorkspaceID: wsID1} ws1 := tailnet.Workspace{ID: wsID1, Name: "ws1"} ws2 := tailnet.Workspace{ID: wsID2, Name: "ws2"} @@ -745,7 +744,7 @@ func TestProcessFreshState(t *testing.T) { initialAgents := map[uuid.UUID]tailnet.Agent{ agentID1: agent1, agentID2: agent2, - agentID4: agent4, // agent 4 is initially present + agentID4: agent4, } tests := []struct { @@ -761,8 +760,8 @@ func TestProcessFreshState(t *testing.T) { FreshState: true, UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, UpsertedAgents: []*tailnet.Agent{&agent1, &agent2, &agent4}, - DeletedWorkspaces: []*tailnet.Workspace{}, // Input deletions - DeletedAgents: []*tailnet.Agent{}, // Input deletions + DeletedWorkspaces: []*tailnet.Workspace{}, + DeletedAgents: []*tailnet.Agent{}, }, expectedDelete: &tailnet.WorkspaceUpdate{ // Expect no *additional* deletions DeletedWorkspaces: []*tailnet.Workspace{}, @@ -852,23 +851,6 @@ func TestProcessFreshState(t *testing.T) { }, }, }, - { - name: "UpdateWithExistingDeletions", // Agent 2 removed, ws2 also removed, but update already contains some deletions - initialAgents: initialAgents, - update: &tailnet.WorkspaceUpdate{ - FreshState: true, - UpsertedWorkspaces: []*tailnet.Workspace{&ws1}, // ws2 not present - UpsertedAgents: []*tailnet.Agent{&agent1, &agent4}, // agent2 not present - DeletedWorkspaces: []*tailnet.Workspace{{ID: uuid.New()}}, // Pre-existing deletion - DeletedAgents: []*tailnet.Agent{{ID: uuid.New()}}, // Pre-existing deletion - }, - expectedDelete: &tailnet.WorkspaceUpdate{ - DeletedWorkspaces: []*tailnet.Workspace{{ID: wsID2}}, // Expect ws2 to be added to deletions - DeletedAgents: []*tailnet.Agent{ // Expect agent2 to be added to deletions - {ID: agentID2, Name: "agent2", WorkspaceID: wsID2}, - }, - }, - }, } for _, tt := range tests { @@ -876,41 +858,13 @@ func TestProcessFreshState(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - // Keep track of original lengths to compare only the added elements - originalDeletedAgentsLen := len(tt.update.DeletedAgents) - originalDeletedWorkspacesLen := len(tt.update.DeletedWorkspaces) - agentsCopy := make(map[uuid.UUID]tailnet.Agent) maps.Copy(agentsCopy, tt.initialAgents) processFreshState(tt.update, agentsCopy) - // Extract only the elements added by processFreshState - addedDeletedAgents := tt.update.DeletedAgents[originalDeletedAgentsLen:] - addedDeletedWorkspaces := tt.update.DeletedWorkspaces[originalDeletedWorkspacesLen:] - - // Sort slices for consistent comparison - sortAgents(addedDeletedAgents) - sortAgents(tt.expectedDelete.DeletedAgents) - sortWorkspaces(addedDeletedWorkspaces) - sortWorkspaces(tt.expectedDelete.DeletedWorkspaces) - - require.ElementsMatch(t, tt.expectedDelete.DeletedAgents, addedDeletedAgents, "DeletedAgents mismatch") - require.ElementsMatch(t, tt.expectedDelete.DeletedWorkspaces, addedDeletedWorkspaces, "DeletedWorkspaces mismatch") + require.ElementsMatch(t, tt.expectedDelete.DeletedAgents, tt.update.DeletedAgents, "DeletedAgents mismatch") + require.ElementsMatch(t, tt.expectedDelete.DeletedWorkspaces, tt.update.DeletedWorkspaces, "DeletedWorkspaces mismatch") }) } } - -// sortAgents sorts a slice of Agents by ID for consistent comparison. -func sortAgents(agents []*tailnet.Agent) { - sort.Slice(agents, func(i, j int) bool { - return agents[i].ID.String() < agents[j].ID.String() - }) -} - -// sortWorkspaces sorts a slice of Workspaces by ID for consistent comparison. -func sortWorkspaces(workspaces []*tailnet.Workspace) { - sort.Slice(workspaces, func(i, j int) bool { - return workspaces[i].ID.String() < workspaces[j].ID.String() - }) -} From 5dec731072d71659a3851a39cabe9d83702669fa Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 5 May 2025 12:26:38 +0000 Subject: [PATCH 07/10] refactored FreshState to UpdateKind to be a better indicator of the field's usage --- tailnet/controllers.go | 23 +++++++++++++++-------- tailnet/controllers_test.go | 15 ++++++++------- vpn/tunnel.go | 14 +++++++++----- vpn/tunnel_internal_test.go | 19 ++++++++++--------- 4 files changed, 42 insertions(+), 29 deletions(-) diff --git a/tailnet/controllers.go b/tailnet/controllers.go index 34d28731f521c..44e051f4a1846 100644 --- a/tailnet/controllers.go +++ b/tailnet/controllers.go @@ -1049,7 +1049,7 @@ func (t *tunnelUpdater) recvLoop() { t.logger.Debug(context.Background(), "tunnel updater recvLoop started") defer t.logger.Debug(context.Background(), "tunnel updater recvLoop done") defer close(t.recvLoopDone) - freshState := true + updateKind := Snapshot for { update, err := t.client.Recv() if err != nil { @@ -1062,10 +1062,10 @@ func (t *tunnelUpdater) recvLoop() { } t.logger.Debug(context.Background(), "got workspace update", slog.F("workspace_update", update), - slog.F("fresh_state", freshState), + slog.F("update_kind", updateKind), ) - err = t.handleUpdate(update, freshState) - freshState = false + err = t.handleUpdate(update, updateKind) + updateKind = Diff if err != nil { t.logger.Critical(context.Background(), "failed to handle workspace Update", slog.Error(err)) cErr := t.client.Close() @@ -1086,16 +1086,23 @@ type WorkspaceUpdate struct { UpsertedAgents []*Agent DeletedWorkspaces []*Workspace DeletedAgents []*Agent - FreshState bool + Kind UpdateKind } +type UpdateKind int + +const ( + Diff UpdateKind = iota + Snapshot +) + func (w *WorkspaceUpdate) Clone() WorkspaceUpdate { clone := WorkspaceUpdate{ UpsertedWorkspaces: make([]*Workspace, len(w.UpsertedWorkspaces)), UpsertedAgents: make([]*Agent, len(w.UpsertedAgents)), DeletedWorkspaces: make([]*Workspace, len(w.DeletedWorkspaces)), DeletedAgents: make([]*Agent, len(w.DeletedAgents)), - FreshState: w.FreshState, + Kind: w.Kind, } for i, ws := range w.UpsertedWorkspaces { clone.UpsertedWorkspaces[i] = &Workspace{ @@ -1120,7 +1127,7 @@ func (w *WorkspaceUpdate) Clone() WorkspaceUpdate { return clone } -func (t *tunnelUpdater) handleUpdate(update *proto.WorkspaceUpdate, freshState bool) error { +func (t *tunnelUpdater) handleUpdate(update *proto.WorkspaceUpdate, updateKind UpdateKind) error { t.Lock() defer t.Unlock() @@ -1129,7 +1136,7 @@ func (t *tunnelUpdater) handleUpdate(update *proto.WorkspaceUpdate, freshState b UpsertedAgents: []*Agent{}, DeletedWorkspaces: []*Workspace{}, DeletedAgents: []*Agent{}, - FreshState: freshState, + Kind: updateKind, } for _, uw := range update.UpsertedWorkspaces { diff --git a/tailnet/controllers_test.go b/tailnet/controllers_test.go index eac6a15326007..31c1210d13c2b 100644 --- a/tailnet/controllers_test.go +++ b/tailnet/controllers_test.go @@ -1611,7 +1611,7 @@ func TestTunnelAllWorkspaceUpdatesController_Initial(t *testing.T) { }, DeletedWorkspaces: []*tailnet.Workspace{}, DeletedAgents: []*tailnet.Agent{}, - FreshState: true, + Kind: tailnet.Snapshot, } // And the callback @@ -1619,7 +1619,6 @@ func TestTunnelAllWorkspaceUpdatesController_Initial(t *testing.T) { require.Equal(t, currentState, cbUpdate) // Current recvState should match but shouldn't be a fresh state - currentState.FreshState = false recvState, err := updateCtrl.CurrentState() require.NoError(t, err) slices.SortFunc(recvState.UpsertedWorkspaces, func(a, b *tailnet.Workspace) int { @@ -1628,6 +1627,9 @@ func TestTunnelAllWorkspaceUpdatesController_Initial(t *testing.T) { slices.SortFunc(recvState.UpsertedAgents, func(a, b *tailnet.Agent) int { return strings.Compare(a.Name, b.Name) }) + // tunnel is still open, so it's a diff + currentState.Kind = tailnet.Diff + require.Equal(t, currentState, recvState) } @@ -1694,16 +1696,17 @@ func TestTunnelAllWorkspaceUpdatesController_DeleteAgent(t *testing.T) { }, DeletedWorkspaces: []*tailnet.Workspace{}, DeletedAgents: []*tailnet.Agent{}, - FreshState: true, + Kind: tailnet.Snapshot, } cbUpdate := testutil.TryReceive(ctx, t, fUH.ch) require.Equal(t, initRecvUp, cbUpdate) - // Current state should match initial but shouldn't be a fresh state - initRecvUp.FreshState = false state, err := updateCtrl.CurrentState() require.NoError(t, err) + // tunnel is still open, so it's a diff + initRecvUp.Kind = tailnet.Diff + require.Equal(t, initRecvUp, state) // Send update that removes w1a1 and adds w1a2 @@ -1757,7 +1760,6 @@ func TestTunnelAllWorkspaceUpdatesController_DeleteAgent(t *testing.T) { "w1.coder.": {ws1a1IP}, }}, }, - FreshState: false, } require.Equal(t, sndRecvUpdate, cbUpdate) @@ -1776,7 +1778,6 @@ func TestTunnelAllWorkspaceUpdatesController_DeleteAgent(t *testing.T) { }, DeletedWorkspaces: []*tailnet.Workspace{}, DeletedAgents: []*tailnet.Agent{}, - FreshState: false, }, state) } diff --git a/vpn/tunnel.go b/vpn/tunnel.go index 4731ad62606d5..d055e91d83b5c 100644 --- a/vpn/tunnel.go +++ b/vpn/tunnel.go @@ -397,9 +397,9 @@ func (u *updater) sendUpdateResponse(req *request[*TunnelMessage, *ManagerMessag // createPeerUpdateLocked creates a PeerUpdate message from a workspace update, populating // the network status of the agents. func (u *updater) createPeerUpdateLocked(update tailnet.WorkspaceUpdate) *PeerUpdate { - // this flag is true on the first update after a reconnect - if update.FreshState { - processFreshState(&update, u.agents) + // if the update is a snapshot, we need to process the full state + if update.Kind == tailnet.Snapshot { + processSnapshotUpdate(&update, u.agents) } out := &PeerUpdate{ @@ -554,8 +554,12 @@ func (u *updater) netStatusLoop() { } } -// processFreshState handles the logic for when a fresh state update is received. -func processFreshState(update *tailnet.WorkspaceUpdate, agents map[uuid.UUID]tailnet.Agent) { +// processSnapshotUpdate handles the logic when a full state update is received. +// When the tunnel is live, we only receive diffs, but the first packet on any given +// reconnect to the tailnet API is a full state. +// Without this logic we weren't processing deletes for any workspaces or agents deleted +// while the client was disconnected while the computer was asleep. +func processSnapshotUpdate(update *tailnet.WorkspaceUpdate, agents map[uuid.UUID]tailnet.Agent) { // ignoredWorkspaces is initially populated with the workspaces that are // in the current update. Later on we populate it with the deleted workspaces too // so that we don't send duplicate updates. Same applies to ignoredAgents. diff --git a/vpn/tunnel_internal_test.go b/vpn/tunnel_internal_test.go index 0dca9ca0be59c..5c423c0645302 100644 --- a/vpn/tunnel_internal_test.go +++ b/vpn/tunnel_internal_test.go @@ -567,7 +567,7 @@ func TestTunnel_sendAgentUpdateReconnect(t *testing.T) { }, }, }, - FreshState: true, + Kind: tailnet.Snapshot, }) require.NoError(t, err) @@ -579,6 +579,7 @@ func TestTunnel_sendAgentUpdateReconnect(t *testing.T) { require.NotNil(t, peerUpdate) require.Len(t, peerUpdate.UpsertedAgents, 1) require.Len(t, peerUpdate.DeletedAgents, 1) + require.Len(t, peerUpdate.DeletedWorkspaces, 0) require.Equal(t, aID2[:], peerUpdate.UpsertedAgents[0].Id) require.Equal(t, hsTime, peerUpdate.UpsertedAgents[0].LastHandshake.AsTime()) @@ -667,7 +668,7 @@ func TestTunnel_sendAgentUpdateWorkspaceReconnect(t *testing.T) { }, }, }, - FreshState: true, + Kind: tailnet.Snapshot, }) require.NoError(t, err) @@ -757,7 +758,7 @@ func TestProcessFreshState(t *testing.T) { name: "NoChange", initialAgents: initialAgents, update: &tailnet.WorkspaceUpdate{ - FreshState: true, + Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, UpsertedAgents: []*tailnet.Agent{&agent1, &agent2, &agent4}, DeletedWorkspaces: []*tailnet.Workspace{}, @@ -772,7 +773,7 @@ func TestProcessFreshState(t *testing.T) { name: "AgentAdded", // Agent 3 added in update initialAgents: initialAgents, update: &tailnet.WorkspaceUpdate{ - FreshState: true, + Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2, &ws3}, UpsertedAgents: []*tailnet.Agent{&agent1, &agent2, &agent3, &agent4}, DeletedWorkspaces: []*tailnet.Workspace{}, @@ -787,7 +788,7 @@ func TestProcessFreshState(t *testing.T) { name: "AgentRemovedWorkspaceAlsoRemoved", // Agent 2 removed, ws2 also removed initialAgents: initialAgents, update: &tailnet.WorkspaceUpdate{ - FreshState: true, + Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{&ws1}, // ws2 not present UpsertedAgents: []*tailnet.Agent{&agent1, &agent4}, // agent2 not present DeletedWorkspaces: []*tailnet.Workspace{}, @@ -804,7 +805,7 @@ func TestProcessFreshState(t *testing.T) { name: "AgentRemovedWorkspaceStays", // Agent 4 removed, but ws1 stays (due to agent1) initialAgents: initialAgents, update: &tailnet.WorkspaceUpdate{ - FreshState: true, + Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, // ws1 still present UpsertedAgents: []*tailnet.Agent{&agent1, &agent2}, // agent4 not present DeletedWorkspaces: []*tailnet.Workspace{}, @@ -821,7 +822,7 @@ func TestProcessFreshState(t *testing.T) { name: "InitialAgentsEmpty", initialAgents: map[uuid.UUID]tailnet.Agent{}, // Start with no agents known update: &tailnet.WorkspaceUpdate{ - FreshState: true, + Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, UpsertedAgents: []*tailnet.Agent{&agent1, &agent2}, DeletedWorkspaces: []*tailnet.Workspace{}, @@ -836,7 +837,7 @@ func TestProcessFreshState(t *testing.T) { name: "UpdateEmpty", // Fresh state says nothing exists initialAgents: initialAgents, update: &tailnet.WorkspaceUpdate{ - FreshState: true, + Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{}, UpsertedAgents: []*tailnet.Agent{}, DeletedWorkspaces: []*tailnet.Workspace{}, @@ -861,7 +862,7 @@ func TestProcessFreshState(t *testing.T) { agentsCopy := make(map[uuid.UUID]tailnet.Agent) maps.Copy(agentsCopy, tt.initialAgents) - processFreshState(tt.update, agentsCopy) + processSnapshotUpdate(tt.update, agentsCopy) require.ElementsMatch(t, tt.expectedDelete.DeletedAgents, tt.update.DeletedAgents, "DeletedAgents mismatch") require.ElementsMatch(t, tt.expectedDelete.DeletedWorkspaces, tt.update.DeletedWorkspaces, "DeletedWorkspaces mismatch") From ae34934ac6ce8de7d66bb11f1d64ed684ea539df Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 5 May 2025 13:04:06 +0000 Subject: [PATCH 08/10] added present workspaces to the tunnel state --- tailnet/controllers.go | 15 +++++++ tailnet/controllers_test.go | 2 +- vpn/tunnel.go | 37 +++++++++------- vpn/tunnel_internal_test.go | 84 +++++++++++++++++++++++++++---------- 4 files changed, 101 insertions(+), 37 deletions(-) diff --git a/tailnet/controllers.go b/tailnet/controllers.go index 44e051f4a1846..57dcf2749ca67 100644 --- a/tailnet/controllers.go +++ b/tailnet/controllers.go @@ -897,6 +897,21 @@ type Workspace struct { agents map[uuid.UUID]*Agent } +func (a *Workspace) Clone() Workspace { + agents := make(map[uuid.UUID]*Agent, len(a.agents)) + for k, v := range a.agents { + clone := v.Clone() + agents[k] = &clone + } + return Workspace{ + ID: a.ID, + Name: a.Name, + Status: a.Status, + ownerUsername: a.ownerUsername, + agents: agents, + } +} + type DNSNameOptions struct { Suffix string } diff --git a/tailnet/controllers_test.go b/tailnet/controllers_test.go index 31c1210d13c2b..bb5b543378ebe 100644 --- a/tailnet/controllers_test.go +++ b/tailnet/controllers_test.go @@ -1618,7 +1618,7 @@ func TestTunnelAllWorkspaceUpdatesController_Initial(t *testing.T) { cbUpdate := testutil.TryReceive(ctx, t, fUH.ch) require.Equal(t, currentState, cbUpdate) - // Current recvState should match but shouldn't be a fresh state + // Current recvState should match recvState, err := updateCtrl.CurrentState() require.NoError(t, err) slices.SortFunc(recvState.UpsertedWorkspaces, func(a, b *tailnet.Workspace) int { diff --git a/vpn/tunnel.go b/vpn/tunnel.go index d055e91d83b5c..6c71aecaa0965 100644 --- a/vpn/tunnel.go +++ b/vpn/tunnel.go @@ -88,6 +88,7 @@ func NewTunnel( netLoopDone: make(chan struct{}), uSendCh: s.sendCh, agents: map[uuid.UUID]tailnet.Agent{}, + workspaces: map[uuid.UUID]tailnet.Workspace{}, clock: quartz.NewReal(), }, } @@ -347,7 +348,9 @@ type updater struct { uSendCh chan<- *TunnelMessage // agents contains the agents that are currently connected to the tunnel. agents map[uuid.UUID]tailnet.Agent - conn Conn + // workspaces contains the workspaces to which agents are currently connected via the tunnel. + workspaces map[uuid.UUID]tailnet.Workspace + conn Conn clock quartz.Clock } @@ -399,7 +402,7 @@ func (u *updater) sendUpdateResponse(req *request[*TunnelMessage, *ManagerMessag func (u *updater) createPeerUpdateLocked(update tailnet.WorkspaceUpdate) *PeerUpdate { // if the update is a snapshot, we need to process the full state if update.Kind == tailnet.Snapshot { - processSnapshotUpdate(&update, u.agents) + processSnapshotUpdate(&update, u.agents, u.workspaces) } out := &PeerUpdate{ @@ -417,6 +420,12 @@ func (u *updater) createPeerUpdateLocked(update tailnet.WorkspaceUpdate) *PeerUp for _, agent := range update.DeletedAgents { delete(u.agents, agent.ID) } + for _, workspace := range update.UpsertedWorkspaces { + u.workspaces[workspace.ID] = workspace.Clone() + } + for _, workspace := range update.DeletedWorkspaces { + delete(u.workspaces, workspace.ID) + } for i, ws := range update.UpsertedWorkspaces { out.UpsertedWorkspaces[i] = &Workspace{ @@ -559,7 +568,7 @@ func (u *updater) netStatusLoop() { // reconnect to the tailnet API is a full state. // Without this logic we weren't processing deletes for any workspaces or agents deleted // while the client was disconnected while the computer was asleep. -func processSnapshotUpdate(update *tailnet.WorkspaceUpdate, agents map[uuid.UUID]tailnet.Agent) { +func processSnapshotUpdate(update *tailnet.WorkspaceUpdate, agents map[uuid.UUID]tailnet.Agent, workspaces map[uuid.UUID]tailnet.Workspace) { // ignoredWorkspaces is initially populated with the workspaces that are // in the current update. Later on we populate it with the deleted workspaces too // so that we don't send duplicate updates. Same applies to ignoredAgents. @@ -573,23 +582,23 @@ func processSnapshotUpdate(update *tailnet.WorkspaceUpdate, agents map[uuid.UUID ignoredAgents[agent.ID] = struct{}{} } for _, agent := range agents { - if _, ok := ignoredAgents[agent.ID]; !ok { + if _, present := ignoredAgents[agent.ID]; !present { // delete any current agents that are not in the new update update.DeletedAgents = append(update.DeletedAgents, &tailnet.Agent{ ID: agent.ID, Name: agent.Name, WorkspaceID: agent.WorkspaceID, }) - // if the workspace connected to an agent we're deleting, - // is not present in the fresh state, add it to the deleted workspaces - if _, ok := ignoredWorkspaces[agent.WorkspaceID]; !ok { - update.DeletedWorkspaces = append(update.DeletedWorkspaces, &tailnet.Workspace{ - // other fields cannot be populated because the tunnel - // only stores agents and corresponding workspaceIDs - ID: agent.WorkspaceID, - }) - ignoredWorkspaces[agent.WorkspaceID] = struct{}{} - } + } + } + for _, workspace := range workspaces { + if _, present := ignoredWorkspaces[workspace.ID]; !present { + update.DeletedWorkspaces = append(update.DeletedWorkspaces, &tailnet.Workspace{ + ID: workspace.ID, + Name: workspace.Name, + Status: workspace.Status, + }) + ignoredWorkspaces[workspace.ID] = struct{}{} } } } diff --git a/vpn/tunnel_internal_test.go b/vpn/tunnel_internal_test.go index 5c423c0645302..2beba66d7a7d2 100644 --- a/vpn/tunnel_internal_test.go +++ b/vpn/tunnel_internal_test.go @@ -293,6 +293,7 @@ func TestUpdater_createPeerUpdate(t *testing.T) { ctx: ctx, netLoopDone: make(chan struct{}), agents: map[uuid.UUID]tailnet.Agent{}, + workspaces: map[uuid.UUID]tailnet.Workspace{}, conn: newFakeConn(tailnet.WorkspaceUpdate{}, hsTime), } @@ -727,6 +728,7 @@ func TestProcessFreshState(t *testing.T) { wsID1 := uuid.New() wsID2 := uuid.New() wsID3 := uuid.New() + wsID4 := uuid.New() agentID1 := uuid.New() agentID2 := uuid.New() @@ -738,25 +740,32 @@ func TestProcessFreshState(t *testing.T) { agent3 := tailnet.Agent{ID: agentID3, Name: "agent3", WorkspaceID: wsID3} agent4 := tailnet.Agent{ID: agentID4, Name: "agent4", WorkspaceID: wsID1} - ws1 := tailnet.Workspace{ID: wsID1, Name: "ws1"} - ws2 := tailnet.Workspace{ID: wsID2, Name: "ws2"} - ws3 := tailnet.Workspace{ID: wsID3, Name: "ws3"} + ws1 := tailnet.Workspace{ID: wsID1, Name: "ws1", Status: proto.Workspace_RUNNING} + ws2 := tailnet.Workspace{ID: wsID2, Name: "ws2", Status: proto.Workspace_RUNNING} + ws3 := tailnet.Workspace{ID: wsID3, Name: "ws3", Status: proto.Workspace_RUNNING} + ws4 := tailnet.Workspace{ID: wsID4, Name: "ws4", Status: proto.Workspace_RUNNING} initialAgents := map[uuid.UUID]tailnet.Agent{ agentID1: agent1, agentID2: agent2, agentID4: agent4, } + initialWorkspaces := map[uuid.UUID]tailnet.Workspace{ + wsID1: ws1, + wsID2: ws2, + } tests := []struct { - name string - initialAgents map[uuid.UUID]tailnet.Agent - update *tailnet.WorkspaceUpdate - expectedDelete *tailnet.WorkspaceUpdate // We only care about deletions added by the function + name string + initialAgents map[uuid.UUID]tailnet.Agent + initialWorkspaces map[uuid.UUID]tailnet.Workspace + update *tailnet.WorkspaceUpdate + expectedDelete *tailnet.WorkspaceUpdate // We only care about deletions added by the function }{ { - name: "NoChange", - initialAgents: initialAgents, + name: "NoChange", + initialAgents: initialAgents, + initialWorkspaces: initialWorkspaces, update: &tailnet.WorkspaceUpdate{ Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, @@ -770,8 +779,9 @@ func TestProcessFreshState(t *testing.T) { }, }, { - name: "AgentAdded", // Agent 3 added in update - initialAgents: initialAgents, + name: "AgentAdded", // Agent 3 added in update + initialAgents: initialAgents, + initialWorkspaces: initialWorkspaces, update: &tailnet.WorkspaceUpdate{ Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2, &ws3}, @@ -785,8 +795,9 @@ func TestProcessFreshState(t *testing.T) { }, }, { - name: "AgentRemovedWorkspaceAlsoRemoved", // Agent 2 removed, ws2 also removed - initialAgents: initialAgents, + name: "AgentRemovedWorkspaceAlsoRemoved", // Agent 2 removed, ws2 also removed + initialAgents: initialAgents, + initialWorkspaces: initialWorkspaces, update: &tailnet.WorkspaceUpdate{ Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{&ws1}, // ws2 not present @@ -795,15 +806,18 @@ func TestProcessFreshState(t *testing.T) { DeletedAgents: []*tailnet.Agent{}, }, expectedDelete: &tailnet.WorkspaceUpdate{ - DeletedWorkspaces: []*tailnet.Workspace{{ID: wsID2}}, // Expect ws2 to be deleted + DeletedWorkspaces: []*tailnet.Workspace{ + {ID: wsID2, Name: "ws2", Status: proto.Workspace_RUNNING}, + }, // Expect ws2 to be deleted DeletedAgents: []*tailnet.Agent{ // Expect agent2 to be deleted {ID: agentID2, Name: "agent2", WorkspaceID: wsID2}, }, }, }, { - name: "AgentRemovedWorkspaceStays", // Agent 4 removed, but ws1 stays (due to agent1) - initialAgents: initialAgents, + name: "AgentRemovedWorkspaceStays", // Agent 4 removed, but ws1 stays (due to agent1) + initialAgents: initialAgents, + initialWorkspaces: initialWorkspaces, update: &tailnet.WorkspaceUpdate{ Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, // ws1 still present @@ -819,8 +833,9 @@ func TestProcessFreshState(t *testing.T) { }, }, { - name: "InitialAgentsEmpty", - initialAgents: map[uuid.UUID]tailnet.Agent{}, // Start with no agents known + name: "InitialAgentsEmpty", + initialAgents: map[uuid.UUID]tailnet.Agent{}, // Start with no agents known + initialWorkspaces: map[uuid.UUID]tailnet.Workspace{}, update: &tailnet.WorkspaceUpdate{ Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, @@ -834,8 +849,9 @@ func TestProcessFreshState(t *testing.T) { }, }, { - name: "UpdateEmpty", // Fresh state says nothing exists - initialAgents: initialAgents, + name: "UpdateEmpty", // Snapshot says nothing exists + initialAgents: initialAgents, + initialWorkspaces: initialWorkspaces, update: &tailnet.WorkspaceUpdate{ Kind: tailnet.Snapshot, UpsertedWorkspaces: []*tailnet.Workspace{}, @@ -844,7 +860,10 @@ func TestProcessFreshState(t *testing.T) { DeletedAgents: []*tailnet.Agent{}, }, expectedDelete: &tailnet.WorkspaceUpdate{ // Expect all initial agents/workspaces to be deleted - DeletedWorkspaces: []*tailnet.Workspace{{ID: wsID1}, {ID: wsID2}}, // ws1 and ws2 deleted + DeletedWorkspaces: []*tailnet.Workspace{ + {ID: wsID1, Name: "ws1", Status: proto.Workspace_RUNNING}, + {ID: wsID2, Name: "ws2", Status: proto.Workspace_RUNNING}, + }, // ws1 and ws2 deleted DeletedAgents: []*tailnet.Agent{ // agent1, agent2, agent4 deleted {ID: agentID1, Name: "agent1", WorkspaceID: wsID1}, {ID: agentID2, Name: "agent2", WorkspaceID: wsID2}, @@ -852,6 +871,24 @@ func TestProcessFreshState(t *testing.T) { }, }, }, + { + name: "WorkspaceWithNoAgents", // Snapshot says nothing exists + initialAgents: initialAgents, + initialWorkspaces: map[uuid.UUID]tailnet.Workspace{wsID1: ws1, wsID2: ws2, wsID4: ws4}, // ws4 has no agents + update: &tailnet.WorkspaceUpdate{ + Kind: tailnet.Snapshot, + UpsertedWorkspaces: []*tailnet.Workspace{&ws1, &ws2}, + UpsertedAgents: []*tailnet.Agent{&agent1, &agent2, &agent4}, + DeletedWorkspaces: []*tailnet.Workspace{}, + DeletedAgents: []*tailnet.Agent{}, + }, + expectedDelete: &tailnet.WorkspaceUpdate{ // Expect all initial agents/workspaces to be deleted + DeletedWorkspaces: []*tailnet.Workspace{ + {ID: wsID4, Name: "ws4", Status: proto.Workspace_RUNNING}, + }, // ws4 should be deleted + DeletedAgents: []*tailnet.Agent{}, + }, + }, } for _, tt := range tests { @@ -862,7 +899,10 @@ func TestProcessFreshState(t *testing.T) { agentsCopy := make(map[uuid.UUID]tailnet.Agent) maps.Copy(agentsCopy, tt.initialAgents) - processSnapshotUpdate(tt.update, agentsCopy) + workspaceCopy := make(map[uuid.UUID]tailnet.Workspace) + maps.Copy(workspaceCopy, tt.initialWorkspaces) + + processSnapshotUpdate(tt.update, agentsCopy, workspaceCopy) require.ElementsMatch(t, tt.expectedDelete.DeletedAgents, tt.update.DeletedAgents, "DeletedAgents mismatch") require.ElementsMatch(t, tt.expectedDelete.DeletedWorkspaces, tt.update.DeletedWorkspaces, "DeletedWorkspaces mismatch") From 101702d0f4cec4bb6fb56852e66c8be9db8eb5bb Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 5 May 2025 13:20:09 +0000 Subject: [PATCH 09/10] fix lint; wrong receiver name --- tailnet/controllers.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tailnet/controllers.go b/tailnet/controllers.go index 57dcf2749ca67..347ca3f295660 100644 --- a/tailnet/controllers.go +++ b/tailnet/controllers.go @@ -897,17 +897,17 @@ type Workspace struct { agents map[uuid.UUID]*Agent } -func (a *Workspace) Clone() Workspace { +func (w *Workspace) Clone() Workspace { agents := make(map[uuid.UUID]*Agent, len(a.agents)) - for k, v := range a.agents { + for k, v := range w.agents { clone := v.Clone() agents[k] = &clone } return Workspace{ - ID: a.ID, - Name: a.Name, - Status: a.Status, - ownerUsername: a.ownerUsername, + ID: w.ID, + Name: w.Name, + Status: w.Status, + ownerUsername: w.ownerUsername, agents: agents, } } From 0d2ecb29170dfd4519534761304a9aadc66521a8 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 5 May 2025 13:26:41 +0000 Subject: [PATCH 10/10] compile error mishap --- tailnet/controllers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tailnet/controllers.go b/tailnet/controllers.go index 347ca3f295660..b7d4e246a4bee 100644 --- a/tailnet/controllers.go +++ b/tailnet/controllers.go @@ -898,7 +898,7 @@ type Workspace struct { } func (w *Workspace) Clone() Workspace { - agents := make(map[uuid.UUID]*Agent, len(a.agents)) + agents := make(map[uuid.UUID]*Agent, len(w.agents)) for k, v := range w.agents { clone := v.Clone() agents[k] = &clone pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy