Skip to content

Commit f7282a1

Browse files
VR: don't drop aggregated commands when two jobs aggregate on one router
VirtualRoutingResource keeps one aggregation queue per router name. When two jobs aggregate commands for the same router at the same time, for example parallel tier deploys in one VPC: - the second Start replaces the first job's queue, so the commands the first job queued so far are never applied, while their answers already reported success; - the first Finish applies the second job's queue and removes it, so the second Finish finds no queue and fails with a NullPointerException on Queue.clear(). The per-router lock map and the aggregation map were also plain HashMaps changed from parallel agent request threads, and the lock was created with a check-then-put, so two threads could get different locks for the same router. - A Start while a queue exists now joins it instead of replacing it. The first Finish applies the commands of both jobs. - A Finish with no queue no longer fails with an NPE. Its commands were applied by the Finish that ran before it, or directly after that, so it returns the result of the last aggregated apply on the router: success, or that apply's failure details. - The maps are synchronized and the lock is created with computeIfAbsent. The aggregation of a single job behaves as before. Signed-off-by: Brad House <bhouse@nexthop.ai>
1 parent e6f193d commit f7282a1

2 files changed

Lines changed: 191 additions & 11 deletions

File tree

‎core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java‎

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.net.InetSocketAddress;
2424
import java.nio.channels.SocketChannel;
2525
import java.util.ArrayList;
26+
import java.util.Collections;
2627
import java.util.HashMap;
2728
import java.util.List;
2829
import java.util.Map;
@@ -89,8 +90,12 @@ public class VirtualRoutingResource {
8990

9091
protected Logger logger = LogManager.getLogger(getClass());
9192
private VirtualRouterDeployer _vrDeployer;
93+
// Agent request threads run in parallel, so these maps are synchronized. The aggregation maps are
94+
// only read and changed for a router while holding that router's lock from _vrLockMap.
9295
private Map<String, Queue<NetworkElementCommand>> _vrAggregateCommandsSet;
93-
protected Map<String, Lock> _vrLockMap = new HashMap<String, Lock>();
96+
// Router name -> failure details of the last aggregated apply on it, if that apply failed
97+
private Map<String, String> _vrAggregateFailures;
98+
protected Map<String, Lock> _vrLockMap = Collections.synchronizedMap(new HashMap<>());
9499

95100
private String _name;
96101
private int _sleep;
@@ -107,13 +112,7 @@ public VirtualRoutingResource(VirtualRouterDeployer deployer) {
107112
public Answer executeRequest(final NetworkElementCommand cmd) {
108113
boolean aggregated = false;
109114
String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
110-
Lock lock;
111-
if (_vrLockMap.containsKey(routerName)) {
112-
lock = _vrLockMap.get(routerName);
113-
} else {
114-
lock = new ReentrantLock();
115-
_vrLockMap.put(routerName, lock);
116-
}
115+
Lock lock = _vrLockMap.computeIfAbsent(routerName, k -> new ReentrantLock());
117116
lock.lock();
118117

119118
try {
@@ -538,7 +537,8 @@ public boolean configure(final String name, final Map<String, Object> params) th
538537
throw new ConfigurationException("Unable to find the resource for VirtualRouterDeployer!");
539538
}
540539

541-
_vrAggregateCommandsSet = new HashMap<>();
540+
_vrAggregateCommandsSet = Collections.synchronizedMap(new HashMap<>());
541+
_vrAggregateFailures = Collections.synchronizedMap(new HashMap<>());
542542
return true;
543543
}
544544

@@ -605,14 +605,31 @@ private Answer execute(AggregationControlCommand cmd) {
605605
assert cmd.getRouterAccessIp() != null;
606606

607607
if (action == Action.Start) {
608-
assert (!_vrAggregateCommandsSet.containsKey(routerName));
609-
608+
if (_vrAggregateCommandsSet.containsKey(routerName)) {
609+
// Another job is aggregating commands for this router. Keep its queue: replacing it would
610+
// drop the commands queued so far, and the first Finish applies the commands of both jobs.
611+
logger.warn("Command aggregation is already in progress for router {}, joining it", routerName);
612+
return new Answer(cmd, true, "Command aggregation already in progress");
613+
}
610614
Queue<NetworkElementCommand> queue = new LinkedBlockingQueue<>();
611615
_vrAggregateCommandsSet.put(routerName, queue);
612616
return new Answer(cmd, true, "Command aggregation started");
613617
} else if (action == Action.Finish) {
614618
Queue<NetworkElementCommand> queue = _vrAggregateCommandsSet.get(routerName);
619+
if (queue == null) {
620+
// A concurrent job joined this aggregation and its Finish has already applied the queued
621+
// commands; commands sent since then were applied directly. Report the result of that apply.
622+
String failure = _vrAggregateFailures.get(routerName);
623+
logger.warn("No command aggregation in progress for router {}, the last aggregated apply {}", routerName,
624+
failure == null ? "succeeded" : "failed: " + failure);
625+
if (failure != null) {
626+
return new Answer(cmd, false, failure);
627+
}
628+
return new Answer(cmd, true, "Aggregated commands already applied");
629+
}
615630
int answerCounts = 0;
631+
boolean applied = false;
632+
String failure = "Failed to apply aggregated commands";
616633
try {
617634
StringBuilder sb = new StringBuilder();
618635
sb.append("#Apache CloudStack Virtual Router Config File\n");
@@ -642,16 +659,24 @@ private Answer execute(AggregationControlCommand cmd) {
642659

643660
ExecutionResult result = applyConfigToVR(cmd.getRouterAccessIp(), fileConfigItem, timeout);
644661
if (!result.isSuccess()) {
662+
failure = StringUtils.defaultIfBlank(result.getDetails(), failure);
645663
return new Answer(cmd, false, result.getDetails());
646664
}
647665

648666
result = applyConfigToVR(cmd.getRouterAccessIp(), scriptConfigItem, timeout);
649667
if (!result.isSuccess()) {
668+
failure = StringUtils.defaultIfBlank(result.getDetails(), failure);
650669
return new Answer(cmd, false, result.getDetails());
651670
}
652671

672+
applied = true;
653673
return new Answer(cmd, true, "Command aggregation finished");
654674
} finally {
675+
if (applied) {
676+
_vrAggregateFailures.remove(routerName);
677+
} else {
678+
_vrAggregateFailures.put(routerName, failure);
679+
}
655680
queue.clear();
656681
_vrAggregateCommandsSet.remove(routerName);
657682
}
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.agent.resource.virtualnetwork;
18+
19+
import static org.junit.Assert.assertEquals;
20+
import static org.junit.Assert.assertFalse;
21+
import static org.junit.Assert.assertTrue;
22+
23+
import java.util.HashMap;
24+
import java.util.List;
25+
26+
import org.joda.time.Duration;
27+
import org.junit.Before;
28+
import org.junit.Test;
29+
import org.junit.runner.RunWith;
30+
import org.mockito.ArgumentCaptor;
31+
import org.mockito.Mock;
32+
import org.mockito.Mockito;
33+
import org.mockito.junit.MockitoJUnitRunner;
34+
35+
import com.cloud.agent.api.Answer;
36+
import com.cloud.agent.api.routing.AggregationControlCommand;
37+
import com.cloud.agent.api.routing.AggregationControlCommand.Action;
38+
import com.cloud.agent.api.routing.NetworkElementCommand;
39+
import com.cloud.agent.api.routing.SetPortForwardingRulesCommand;
40+
import com.cloud.agent.api.to.PortForwardingRuleTO;
41+
import com.cloud.utils.ExecutionResult;
42+
43+
@RunWith(MockitoJUnitRunner.class)
44+
public class VirtualRoutingResourceAggregationTest {
45+
46+
private static final String ROUTER_IP = "169.254.3.4";
47+
private static final String ROUTER_GUEST_IP = "10.200.1.1";
48+
private static final String ROUTER_NAME = "r-4-VM";
49+
50+
@Mock
51+
private VirtualRouterDeployer deployer;
52+
53+
private VirtualRoutingResource resource;
54+
55+
@Before
56+
public void setup() throws Exception {
57+
Mockito.when(deployer.prepareCommand(Mockito.any())).thenAnswer(invocation -> {
58+
NetworkElementCommand cmd = invocation.getArgument(0);
59+
cmd.setRouterAccessIp(ROUTER_IP);
60+
return new ExecutionResult(true, null);
61+
});
62+
Mockito.lenient().when(deployer.cleanupCommand(Mockito.any())).thenReturn(new ExecutionResult(true, null));
63+
Mockito.lenient().when(deployer.createFileInVR(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
64+
.thenReturn(new ExecutionResult(true, null));
65+
Mockito.lenient().when(deployer.executeInVR(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any(Duration.class)))
66+
.thenReturn(new ExecutionResult(true, null));
67+
resource = new VirtualRoutingResource(deployer);
68+
resource.configure("VRResource", new HashMap<>());
69+
}
70+
71+
private Answer execute(NetworkElementCommand cmd) {
72+
return resource.executeRequest(cmd);
73+
}
74+
75+
private AggregationControlCommand control(Action action) {
76+
return new AggregationControlCommand(action, ROUTER_NAME, ROUTER_IP, ROUTER_GUEST_IP);
77+
}
78+
79+
private SetPortForwardingRulesCommand portForwarding(String publicIp) {
80+
SetPortForwardingRulesCommand cmd = new SetPortForwardingRulesCommand(List.of(
81+
new PortForwardingRuleTO(1, publicIp, 22, 22, "10.10.1.10", 22, 22, "TCP", false, false)));
82+
cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTER_NAME);
83+
return cmd;
84+
}
85+
86+
private List<String> appliedConfigFiles() {
87+
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
88+
Mockito.verify(deployer, Mockito.atLeast(0)).createFileInVR(Mockito.eq(ROUTER_IP), Mockito.anyString(), Mockito.anyString(), content.capture());
89+
return content.getAllValues();
90+
}
91+
92+
@Test
93+
public void overlappingAggregationsKeepCommandsOfBothJobs() {
94+
assertTrue(execute(control(Action.Start)).getResult());
95+
assertTrue(execute(portForwarding("64.1.1.10")).getResult());
96+
assertTrue(execute(control(Action.Start)).getResult());
97+
assertTrue(execute(portForwarding("64.1.1.20")).getResult());
98+
99+
assertTrue(execute(control(Action.Finish)).getResult());
100+
List<String> files = appliedConfigFiles();
101+
assertEquals(1, files.size());
102+
assertTrue(files.get(0).contains("64.1.1.10"));
103+
assertTrue(files.get(0).contains("64.1.1.20"));
104+
105+
Answer second = execute(control(Action.Finish));
106+
assertTrue(second.getResult());
107+
assertEquals(1, appliedConfigFiles().size());
108+
}
109+
110+
@Test
111+
public void finishOfJoinedAggregationReportsFailureOfTheApply() {
112+
Mockito.when(deployer.executeInVR(Mockito.anyString(), Mockito.eq(VRScripts.VR_CFG), Mockito.anyString(), Mockito.any(Duration.class)))
113+
.thenReturn(new ExecutionResult(false, "vr_cfg.sh failed"));
114+
115+
execute(control(Action.Start));
116+
execute(portForwarding("64.1.1.10"));
117+
execute(control(Action.Start));
118+
execute(portForwarding("64.1.1.20"));
119+
120+
Answer first = execute(control(Action.Finish));
121+
assertFalse(first.getResult());
122+
Answer second = execute(control(Action.Finish));
123+
assertFalse(second.getResult());
124+
assertEquals("vr_cfg.sh failed", second.getDetails());
125+
}
126+
127+
@Test
128+
public void laterSuccessfulApplyClearsRecordedFailure() {
129+
Mockito.when(deployer.executeInVR(Mockito.anyString(), Mockito.eq(VRScripts.VR_CFG), Mockito.anyString(), Mockito.any(Duration.class)))
130+
.thenReturn(new ExecutionResult(false, "vr_cfg.sh failed"), new ExecutionResult(true, null));
131+
132+
execute(control(Action.Start));
133+
assertFalse(execute(control(Action.Finish)).getResult());
134+
execute(control(Action.Start));
135+
assertTrue(execute(control(Action.Finish)).getResult());
136+
137+
assertTrue(execute(control(Action.Finish)).getResult());
138+
}
139+
140+
@Test
141+
public void finishWithoutStartDoesNotFail() {
142+
Answer answer = execute(control(Action.Finish));
143+
assertTrue(answer.getResult());
144+
assertTrue(appliedConfigFiles().isEmpty());
145+
}
146+
147+
@Test
148+
public void commandsAfterFinishAreAppliedDirectly() {
149+
execute(control(Action.Start));
150+
execute(control(Action.Finish));
151+
152+
assertTrue(execute(portForwarding("64.1.1.30")).getResult());
153+
Mockito.verify(deployer, Mockito.atLeastOnce()).executeInVR(Mockito.eq(ROUTER_IP), Mockito.eq(VRScripts.UPDATE_CONFIG), Mockito.anyString(), Mockito.any(Duration.class));
154+
}
155+
}

0 commit comments

Comments
 (0)