Skip to content

Commit 4150e40

Browse files
committed
linstor: consider all diskful copies when picking a host
getDiskfulStoragePools only ever returned the storage pool of a single node, so an endpoint was only looked for on that one node - if its cloudstack host was down or disabled the operation fell back to a temporary resource or a diskless attach although other diskful copies were usable. Return all diskful storage pools ordered by how suited the copy is (in use, then active, then inactive) and walk that order when picking a host. The order matters: thick LVM snapshots on shared storage pools must be read on the node the volume is active on, so the host selection no longer shuffles candidates.
1 parent be4f1ef commit 4150e40

3 files changed

Lines changed: 119 additions & 43 deletions

File tree

‎plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -877,15 +877,17 @@ private Optional<RemoteHostEndPoint> getLinstorEP(DevelopersApi api, StoragePool
877877

878878
private Host getDiskfullHost(DevelopersApi api, StoragePool storagePool, String rscName)
879879
throws ApiException {
880-
List<com.linbit.linstor.api.model.StoragePool> linSPs = LinstorUtil.getDiskfulStoragePools(api, rscName);
881-
if (linSPs != null) {
882-
List<String> linstorNodeNames = linSPs.stream()
883-
.map(com.linbit.linstor.api.model.StoragePool::getNodeName)
884-
.collect(Collectors.toList());
885-
Host host = getEnabledClusterHost(storagePool, linstorNodeNames);
880+
// Take the first diskful copy whose host can be used: the copies are ordered by how
881+
// suited they are (in use, then active, then inactive), and for shared storage pools
882+
// only the active node may be read, so the order must not be broken here.
883+
for (com.linbit.linstor.api.model.StoragePool linSP
884+
: LinstorUtil.getDiskfulStoragePoolsByPreference(api, rscName)) {
885+
Host host = getEnabledClusterHost(storagePool, Collections.singletonList(linSP.getNodeName()));
886886
if (host != null) {
887887
return host;
888888
}
889+
logger.debug("Linstor: no usable cloudstack host for diskful node {}, trying next copy",
890+
linSP.getNodeName());
889891
}
890892
logger.error("Linstor: No diskfull host found.");
891893
return null;

‎plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java‎

Lines changed: 50 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import java.util.Arrays;
4646
import java.util.Collection;
4747
import java.util.Collections;
48+
import java.util.Comparator;
4849
import java.util.HashSet;
4950
import java.util.List;
5051
import java.util.Map;
@@ -195,8 +196,26 @@ public static List<String> getLinstorNodeNames(@Nonnull DevelopersApi api) throw
195196
return nodes.stream().map(Node::getName).collect(Collectors.toList());
196197
}
197198

199+
/**
200+
* How suitable a diskful resource is for reading its data (snapshots, volume copies):
201+
* the node it is in use on is best, then nodes with an active resource, then inactive ones.
202+
* Thick LVM snapshots on shared storage pools (dm-snapshot) are not cluster aware, so they
203+
* must be read on the node the origin volume is active on - callers must honor this order.
204+
*/
205+
private static int diskfulPreference(ResourceWithVolumes rwv) {
206+
if (rwv.getState() != null && Boolean.TRUE.equals(rwv.getState().isInUse())) {
207+
return 2;
208+
}
209+
return rwv.getFlags() == null || !rwv.getFlags().contains(ApiConsts.FLAG_RSC_INACTIVE) ? 1 : 0;
210+
}
211+
212+
/**
213+
* All storage pools holding a diskful copy of the resource, best suited first
214+
* (see {@link #diskfulPreference}). Callers that need a single pool should take the first
215+
* one, callers picking a host should walk the list in order and use the first usable one.
216+
*/
198217
public static List<com.linbit.linstor.api.model.StoragePool>
199-
getDiskfulStoragePools(@Nonnull DevelopersApi api, @Nonnull String rscName) throws ApiException
218+
getDiskfulStoragePoolsByPreference(@Nonnull DevelopersApi api, @Nonnull String rscName) throws ApiException
200219
{
201220
List<ResourceWithVolumes> resources = api.viewResources(
202221
Collections.emptyList(),
@@ -206,56 +225,50 @@ public static List<String> getLinstorNodeNames(@Nonnull DevelopersApi api) throw
206225
null,
207226
null);
208227

209-
String nodeName = null;
210-
String storagePoolName = null;
211-
int bestScore = -1;
212-
for (ResourceWithVolumes rwv : resources) {
213-
if (rwv.getVolumes().isEmpty()) {
214-
continue;
215-
}
216-
Volume vol = rwv.getVolumes().get(0);
217-
if (vol.getProviderKind() == ProviderKind.DISKLESS) {
218-
continue;
219-
}
220-
// Prefer the in-use node, then nodes with an active (not INACTIVE) resource:
221-
// thick LVM snapshots on shared storage pools (dm-snapshot) are not cluster aware
222-
// and must be read on the node the origin volume is active on.
223-
boolean inUse = rwv.getState() != null && Boolean.TRUE.equals(rwv.getState().isInUse());
224-
boolean active = rwv.getFlags() == null || !rwv.getFlags().contains(ApiConsts.FLAG_RSC_INACTIVE);
225-
int score = inUse ? 2 : (active ? 1 : 0);
226-
if (score > bestScore) {
227-
bestScore = score;
228-
nodeName = rwv.getNodeName();
229-
storagePoolName = vol.getStoragePoolName();
230-
if (inUse) {
231-
break;
232-
}
233-
}
234-
}
228+
// node name -> storage pool name, ordered by how suited the copy is
229+
List<Pair<String, String>> candidates = resources.stream()
230+
.filter(rwv -> !rwv.getVolumes().isEmpty()
231+
&& rwv.getVolumes().get(0).getProviderKind() != ProviderKind.DISKLESS)
232+
.sorted(Comparator.comparingInt(LinstorUtil::diskfulPreference).reversed())
233+
.map(rwv -> new Pair<>(rwv.getNodeName(), rwv.getVolumes().get(0).getStoragePoolName()))
234+
.collect(Collectors.toList());
235235

236-
if (nodeName == null) {
237-
return null;
236+
if (candidates.isEmpty()) {
237+
return Collections.emptyList();
238238
}
239239

240240
List<com.linbit.linstor.api.model.StoragePool> sps = api.viewStoragePools(
241-
Collections.singletonList(nodeName),
242-
Collections.singletonList(storagePoolName),
241+
candidates.stream().map(Pair::first).distinct().collect(Collectors.toList()),
242+
candidates.stream().map(Pair::second).distinct().collect(Collectors.toList()),
243243
Collections.emptyList(),
244244
null,
245245
null,
246246
true
247247
);
248-
return sps != null ? sps : Collections.emptyList();
248+
if (sps == null) {
249+
return Collections.emptyList();
250+
}
251+
252+
// viewStoragePools does not keep our ordering, so re-apply it
253+
List<com.linbit.linstor.api.model.StoragePool> ordered = new ArrayList<>();
254+
for (Pair<String, String> candidate : candidates) {
255+
sps.stream()
256+
.filter(sp -> candidate.first().equals(sp.getNodeName())
257+
&& candidate.second().equals(sp.getStoragePoolName()))
258+
.findFirst()
259+
.ifPresent(ordered::add);
260+
}
261+
return ordered;
249262
}
250263

264+
/**
265+
* The storage pool of the best suited diskful copy, null if the resource has none.
266+
*/
251267
public static com.linbit.linstor.api.model.StoragePool
252268
getDiskfulStoragePool(@Nonnull DevelopersApi api, @Nonnull String rscName) throws ApiException
253269
{
254-
List<com.linbit.linstor.api.model.StoragePool> sps = getDiskfulStoragePools(api, rscName);
255-
if (sps != null) {
256-
return !sps.isEmpty() ? sps.get(0) : null;
257-
}
258-
return null;
270+
List<com.linbit.linstor.api.model.StoragePool> sps = getDiskfulStoragePoolsByPreference(api, rscName);
271+
return !sps.isEmpty() ? sps.get(0) : null;
259272
}
260273

261274
public static String getSnapshotPath(com.linbit.linstor.api.model.StoragePool sp, String rscName, String snapshotName) {

‎plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@
1616
// under the License.
1717
package org.apache.cloudstack.storage.datastore.util;
1818

19+
import com.linbit.linstor.api.ApiConsts;
20+
import com.linbit.linstor.api.model.ResourceState;
21+
import com.linbit.linstor.api.model.ResourceWithVolumes;
22+
import com.linbit.linstor.api.model.Volume;
1923
import com.linbit.linstor.api.ApiException;
2024
import com.linbit.linstor.api.DevelopersApi;
2125
import com.linbit.linstor.api.model.AutoSelectFilter;
@@ -166,6 +170,63 @@ public void testGetCapacityStoragePoolsCountsSharedSpaceOnce() throws ApiExcepti
166170
Assert.assertEquals(52428800L * 1024, LinstorUtil.getUsedCapacityBytes(api, "shared"));
167171
}
168172

173+
private ResourceWithVolumes mockResource(String node, String pool, boolean inUse, boolean inactive) {
174+
ResourceWithVolumes rwv = new ResourceWithVolumes();
175+
rwv.setName("cs-test");
176+
rwv.setNodeName(node);
177+
Volume vol = new Volume();
178+
vol.setProviderKind(ProviderKind.LVM);
179+
vol.setStoragePoolName(pool);
180+
rwv.setVolumes(Collections.singletonList(vol));
181+
ResourceState state = new ResourceState();
182+
state.setInUse(inUse);
183+
rwv.setState(state);
184+
if (inactive) {
185+
rwv.setFlags(Collections.singletonList(ApiConsts.FLAG_RSC_INACTIVE));
186+
}
187+
return rwv;
188+
}
189+
190+
@Test
191+
public void testDiskfulStoragePoolsOrderedByPreference() throws ApiException {
192+
// inactive copy first, in-use copy last: the result must be in-use, active, inactive
193+
when(api.viewResources(Collections.emptyList(), Collections.singletonList("cs-test"),
194+
Collections.emptyList(), Collections.emptyList(), null, null))
195+
.thenReturn(Arrays.asList(
196+
mockResource("nodeC", "poolC", false, true),
197+
mockResource("nodeB", "poolB", false, false),
198+
mockResource("nodeA", "poolA", true, false)));
199+
// the pools are queried in preference order (in use, active, inactive)
200+
when(api.viewStoragePools(Arrays.asList("nodeA", "nodeB", "nodeC"),
201+
Arrays.asList("poolA", "poolB", "poolC"), Collections.emptyList(), null, null, true))
202+
.thenReturn(Arrays.asList(
203+
mockStoragePool("poolB", "nodeB", ProviderKind.LVM),
204+
mockStoragePool("poolA", "nodeA", ProviderKind.LVM),
205+
mockStoragePool("poolC", "nodeC", ProviderKind.LVM)));
206+
207+
List<StoragePool> pools = LinstorUtil.getDiskfulStoragePoolsByPreference(api, "cs-test");
208+
Assert.assertEquals(Arrays.asList("nodeA", "nodeB", "nodeC"),
209+
pools.stream().map(StoragePool::getNodeName).collect(Collectors.toList()));
210+
// the single-pool accessor keeps returning the best suited copy
211+
Assert.assertEquals("nodeA", LinstorUtil.getDiskfulStoragePool(api, "cs-test").getNodeName());
212+
}
213+
214+
@Test
215+
public void testDiskfulStoragePoolsIgnoresDiskless() throws ApiException {
216+
ResourceWithVolumes diskless = new ResourceWithVolumes();
217+
diskless.setName("cs-test");
218+
diskless.setNodeName("nodeD");
219+
Volume dlVol = new Volume();
220+
dlVol.setProviderKind(ProviderKind.DISKLESS);
221+
diskless.setVolumes(Collections.singletonList(dlVol));
222+
when(api.viewResources(Collections.emptyList(), Collections.singletonList("cs-test"),
223+
Collections.emptyList(), Collections.emptyList(), null, null))
224+
.thenReturn(Collections.singletonList(diskless));
225+
226+
Assert.assertTrue(LinstorUtil.getDiskfulStoragePoolsByPreference(api, "cs-test").isEmpty());
227+
Assert.assertNull(LinstorUtil.getDiskfulStoragePool(api, "cs-test"));
228+
}
229+
169230
@Test
170231
public void testThickProviderKindDetection() {
171232
Assert.assertTrue(LinstorUtil.isThicklyProvisionedPools(Arrays.asList(

0 commit comments

Comments
 (0)