Skip to content

Commit

Permalink
Add UTs
Browse files Browse the repository at this point in the history 
Signed-off-by: Ashish Singh <[email protected]>
  • Loading branch information
@ashking94
ashking94 committed Aug 27, 2024
1 parent e21d937 commit 1975071
Show file tree
Hide file tree
Showing 6 changed files with 241 additions and 114 deletions.
4 changes: 4 additions & 0 deletions server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ BlobPath hashPath() {
return BlobPath.cleanPath().add(shardId).add(indexUUID());
}

public String shardId() {
return shardId;
}

/**
* Returns a new builder for {@link SnapshotShardPathInput}.
*/
Expand Down
17 changes: 12 additions & 5 deletions server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2076,17 +2076,24 @@ private Set<String> writeNewIndexShardPaths(
private String writeIndexShardPaths(IndexId indexId, SnapshotId snapshotId, int shardCount) {
try {
List<String> paths = getShardPaths(indexId, shardCount);
String pathType = String.valueOf(indexId.getShardPathType());
String pathHashAlgorithm = String.valueOf(FNV_1A_COMPOSITE_1.getCode());
int pathType = indexId.getShardPathType();
int pathHashAlgorithm = FNV_1A_COMPOSITE_1.getCode();
String blobName = String.join(
SnapshotShardPaths.DELIMITER,
indexId.getId(),
indexId.getName(),
String.valueOf(shardCount),
pathType,
pathHashAlgorithm
String.valueOf(pathType),
String.valueOf(pathHashAlgorithm)
);
SnapshotShardPaths shardPaths = new SnapshotShardPaths(
paths,
indexId.getId(),
indexId.getName(),
shardCount,
PathType.fromCode(pathType),
PathHashAlgorithm.fromCode(pathHashAlgorithm)
);
SnapshotShardPaths shardPaths = new SnapshotShardPaths(paths);
SNAPSHOT_SHARD_PATHS_FORMAT.writeAsyncWithUrgentPriority(
shardPaths,
snapshotShardPathBlobContainer(),
Expand Down
89 changes: 41 additions & 48 deletions server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,13 @@

package org.opensearch.snapshots;

import com.fasterxml.jackson.core.JsonParseException;

import org.opensearch.OpenSearchParseException;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
import org.opensearch.index.remote.RemoteStoreEnums.PathType;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
Expand All @@ -34,19 +31,49 @@ public class SnapshotShardPaths implements ToXContent {
public static final String FILE_NAME_FORMAT = "%s";

private static final String PATHS_FIELD = "paths";
private static final String INDEX_ID_FIELD = "indexId";
private static final String INDEX_NAME_FIELD = "indexName";
private static final String NUMBER_OF_SHARDS_FIELD = "number_of_shards";
private static final String SHARD_PATH_TYPE_FIELD = "shard_path_type";
private static final String SHARD_PATH_HASH_ALGORITHM_FIELD = "shard_path_hash_algorithm";

private final List<String> paths;

public SnapshotShardPaths(List<String> paths) {
this.paths = Collections.unmodifiableList(paths);
}

public List<String> getPaths() {
return paths;
private final String indexId;
private final String indexName;
private final int numberOfShards;
private final PathType shardPathType;
private final PathHashAlgorithm shardPathHashAlgorithm;

public SnapshotShardPaths(
List<String> paths,
String indexId,
String indexName,
int numberOfShards,
PathType shardPathType,
PathHashAlgorithm shardPathHashAlgorithm
) {
assert !paths.isEmpty() : "paths must not be empty";
assert indexId != null && !indexId.isEmpty() : "indexId must not be empty";
assert indexName != null && !indexName.isEmpty() : "indexName must not be empty";
assert numberOfShards > 0 : "numberOfShards must be > 0";
assert shardPathType != null : "shardPathType must not be null";
assert shardPathHashAlgorithm != null : "shardPathHashAlgorithm must not be null";

this.paths = paths;
this.indexId = indexId;
this.indexName = indexName;
this.numberOfShards = numberOfShards;
this.shardPathType = shardPathType;
this.shardPathHashAlgorithm = shardPathHashAlgorithm;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(INDEX_ID_FIELD, indexId);
builder.field(INDEX_NAME_FIELD, indexName);
builder.field(NUMBER_OF_SHARDS_FIELD, numberOfShards);
builder.field(SHARD_PATH_TYPE_FIELD, shardPathType.getCode());
builder.field(SHARD_PATH_HASH_ALGORITHM_FIELD, shardPathHashAlgorithm.getCode());
builder.startArray(PATHS_FIELD);
for (String path : paths) {
builder.value(path);
Expand All @@ -55,41 +82,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
return builder;
}

public static SnapshotShardPaths fromXContent(XContentParser parser) throws IOException {
List<String> paths = new ArrayList<>();

try {
XContentParser.Token token = parser.currentToken();
if (token == null) {
token = parser.nextToken();
}

if (token != XContentParser.Token.START_OBJECT) {
throw new OpenSearchParseException("Expected a start object");
}

token = parser.nextToken();
if (token == XContentParser.Token.END_OBJECT) {
throw new OpenSearchParseException("Missing [" + PATHS_FIELD + "] field");
}

while (token != XContentParser.Token.END_OBJECT) {
String fieldName = parser.currentName();
if (PATHS_FIELD.equals(fieldName)) {
if (parser.nextToken() != XContentParser.Token.START_ARRAY) {
throw new OpenSearchParseException("Expected an array for field [" + PATHS_FIELD + "]");
}
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
paths.add(parser.text());
}
} else {
throw new OpenSearchParseException("Unexpected field [" + fieldName + "]");
}
token = parser.nextToken();
}
} catch (JsonParseException e) {
throw new OpenSearchParseException("Failed to parse SnapshotIndexIdPaths", e);
}
return new SnapshotShardPaths(paths);
public static SnapshotShardPaths fromXContent(XContentParser ignored) {
throw new UnsupportedOperationException("SnapshotShardPaths.fromXContent() is not supported");
}
}
42 changes: 42 additions & 0 deletions server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.opensearch.index.remote.RemoteStoreEnums.DataType;
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
import org.opensearch.index.remote.RemoteStorePathStrategy.ShardDataPathInput;
import org.opensearch.index.remote.RemoteStorePathStrategy.SnapshotShardPathInput;
import org.opensearch.test.OpenSearchTestCase;

import java.util.ArrayList;
Expand Down Expand Up @@ -597,6 +598,47 @@ public void testGeneratePathForHashedInfixType() {
assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
}

public void testGeneratePathForSnapshotShardPathInput() {
BlobPath blobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij");
String indexUUID = "dsdkjsu8832njn";
String shardId = "10";
SnapshotShardPathInput pathInput = SnapshotShardPathInput.builder()
.basePath(blobPath)
.indexUUID(indexUUID)
.shardId(shardId)
.build();

// FIXED PATH
BlobPath result = FIXED.path(pathInput, null);
String expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/indices/dsdkjsu8832njn/10/";
String actual = result.buildAsString();
assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected);

// HASHED_PREFIX - FNV_1A_COMPOSITE_1
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
expected = "_11001000010110/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/indices/dsdkjsu8832njn/10/";
actual = result.buildAsString();
assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected);

// HASHED_PREFIX - FNV_1A_BASE64
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
expected = "_yFiSl_VGGM/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/indices/dsdkjsu8832njn/10/";
actual = result.buildAsString();
assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected);

// HASHED_INFIX - FNV_1A_COMPOSITE_1
result = HASHED_INFIX.path(pathInput, FNV_1A_COMPOSITE_1);
expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/_11001000010110/indices/dsdkjsu8832njn/10/";
actual = result.buildAsString();
assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected);

// HASHED_INFIX - FNV_1A_BASE64
result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/_yFiSl_VGGM/indices/dsdkjsu8832njn/10/";
actual = result.buildAsString();
assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected);
}

private String derivePath(String basePath, ShardDataPathInput pathInput) {
return "".equals(basePath)
? String.join(
Expand Down
40 changes: 40 additions & 0 deletions server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,46 @@ public void testFixedSubPath() {
.dataType(DATA)
.build();
assertEquals(BlobPath.cleanPath().add(INDEX_UUID).add(SHARD_ID).add(TRANSLOG.getName()).add(DATA.getName()), input2.fixedSubPath());
}

public void testSnapshotShardPathInput() {
assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.SnapshotShardPathInput.builder().build());
assertThrows(
NullPointerException.class,
() -> RemoteStorePathStrategy.SnapshotShardPathInput.builder().basePath(BASE_PATH).build()
);
assertThrows(
NullPointerException.class,
() -> RemoteStorePathStrategy.SnapshotShardPathInput.builder().indexUUID(INDEX_UUID).build()
);
assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.SnapshotShardPathInput.builder().shardId(SHARD_ID).build());

RemoteStorePathStrategy.SnapshotShardPathInput input = RemoteStorePathStrategy.SnapshotShardPathInput.builder()
.basePath(BASE_PATH)
.indexUUID(INDEX_UUID)
.shardId(SHARD_ID)
.build();
assertEquals(BASE_PATH, input.basePath());
assertEquals(INDEX_UUID, input.indexUUID());
assertEquals(SHARD_ID, input.shardId());
}

public void testSnapshotShardPathInputFixedSubPath() {
RemoteStorePathStrategy.SnapshotShardPathInput input = RemoteStorePathStrategy.SnapshotShardPathInput.builder()
.basePath(BASE_PATH)
.indexUUID(INDEX_UUID)
.shardId(SHARD_ID)
.build();
assertEquals(BlobPath.cleanPath().add("indices").add(INDEX_UUID).add(SHARD_ID), input.fixedSubPath());
}

public void testSnapshotShardPathInputHashPath() {
RemoteStorePathStrategy.SnapshotShardPathInput input = RemoteStorePathStrategy.SnapshotShardPathInput.builder()
.basePath(BASE_PATH)
.indexUUID(INDEX_UUID)
.shardId(SHARD_ID)
.build();
assertEquals(BlobPath.cleanPath().add(SHARD_ID).add(INDEX_UUID), input.hashPath());
}

}
Loading

0 comments on commit 1975071

Please sign in to comment.