From 909a85b7f069336c89471bc8aa94bfd464086563 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Tue, 27 Aug 2024 14:34:25 -0400 Subject: [PATCH 001/108] Adds a new plugin type named ResourcePlugin and relevant base classes Signed-off-by: Darshit Chanpura --- .../accesscontrol/resources/CreatedBy.java | 48 +++++++++ .../accesscontrol/resources/Resource.java | 101 ++++++++++++++++++ .../resources/ResourceSharing.java | 61 +++++++++++ .../accesscontrol/resources/ShareWith.java | 23 ++++ .../opensearch/plugins/ResourcePlugin.java | 73 +++++++++++++ 5 files changed, 306 insertions(+) create mode 100644 server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java create mode 100644 server/src/main/java/org/opensearch/accesscontrol/resources/Resource.java create mode 100644 server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java create mode 100644 server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java create mode 100644 server/src/main/java/org/opensearch/plugins/ResourcePlugin.java diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java b/server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java new file mode 100644 index 0000000000000..01c61e82ed5f4 --- /dev/null +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.accesscontrol.resources; + +/** + * This class contains information on the creator of a resource. + * Creator can either be a user or a backend_role. + * + * @opensearch.experimental + */ +public class CreatedBy { + + private String user; + + private String backendRole; + + public CreatedBy(String user, String backendRole) { + this.user = user; + this.backendRole = backendRole; + } + + public String getBackendRole() { + return backendRole; + } + + public void setBackendRole(String backendRole) { + this.backendRole = backendRole; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + @Override + public String toString() { + return "CreatedBy {" + "user='" + user + '\'' + ", backendRole='" + backendRole + '\'' + '}'; + } +} diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/Resource.java b/server/src/main/java/org/opensearch/accesscontrol/resources/Resource.java new file mode 100644 index 0000000000000..4212cb630cced --- /dev/null +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/Resource.java @@ -0,0 +1,101 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.accesscontrol.resources; + +import java.util.Objects; + +/** + * A document in .resource_sharing index. + * Holds information about the resource (obtained from defining plugin's meta-data), + * the index which defines the resources, the creator of the resource, + * and the information on whom this resource is shared with. + * + * @opensearch.experimental + */ +public class Resource { + + private String sourceIdx; + + private String resourceId; + + private CreatedBy createdBy; + + private ResourceSharing sharedWith; + + public Resource(String sourceIdx, String resourceId, CreatedBy createdBy, ResourceSharing sharedWith) { + this.sourceIdx = sourceIdx; + this.resourceId = resourceId; + this.createdBy = createdBy; + this.sharedWith = sharedWith; + } + + public String getSourceIdx() { + return sourceIdx; + } + + public void setSourceIdx(String sourceIdx) { + this.sourceIdx = sourceIdx; + } + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public CreatedBy getCreatedBy() { + return createdBy; + } + + public void setCreatedBy(CreatedBy createdBy) { + this.createdBy = createdBy; + } + + public ResourceSharing getSharedWith() { + return sharedWith; + } + + public void setSharedWith(ResourceSharing sharedWith) { + this.sharedWith = sharedWith; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Resource resource = (Resource) o; + return Objects.equals(getSourceIdx(), resource.getSourceIdx()) + && Objects.equals(getResourceId(), resource.getResourceId()) + && Objects.equals(getCreatedBy(), resource.getCreatedBy()) + && Objects.equals(getSharedWith(), resource.getSharedWith()); + } + + @Override + public int hashCode() { + return Objects.hash(getSourceIdx(), getResourceId(), getCreatedBy(), getSharedWith()); + } + + @Override + public String toString() { + return "Resource {" + + "sourceIdx='" + + sourceIdx + + '\'' + + ", resourceId='" + + resourceId + + '\'' + + ", createdBy=" + + createdBy + + ", sharedWith=" + + sharedWith + + '}'; + } +} diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java new file mode 100644 index 0000000000000..331ccb5de898c --- /dev/null +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.accesscontrol.resources; + +import java.util.List; + +/** + * This class contains information about whom a resource is shared with. + * It could be a user-name, a role or a backend_role. + * + * @opensearch.experimental + */ +public class ResourceSharing { + + private List users; + + private List roles; + + private List backendRoles; + + public ResourceSharing(List users, List backendRoles, List roles) { + this.users = users; + this.backendRoles = backendRoles; + this.roles = roles; + } + + public List getUsers() { + return users; + } + + public void setUsers(List users) { + this.users = users; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public List getBackendRoles() { + return backendRoles; + } + + public void setBackendRoles(List backendRoles) { + this.backendRoles = backendRoles; + } + + @Override + public String toString() { + return "ResourceSharing {" + "users=" + users + ", roles=" + roles + ", backendRoles=" + backendRoles + '}'; + } +} diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java new file mode 100644 index 0000000000000..702081ebd9ec9 --- /dev/null +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.accesscontrol.resources; + +/** + * This enum contains the type of entities a resource can be shared with. + * + * @opensearch.experimental + */ +public enum ShareWith { + + USERS, + + ROLES, + + BACKEND_ROLES, +} diff --git a/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java b/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java new file mode 100644 index 0000000000000..3a357fd16c4a9 --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.accesscontrol.resources.Resource; +import org.opensearch.accesscontrol.resources.ShareWith; + +import java.util.List; +import java.util.Map; + +/** + * This plugin class defines usage mechanisms for plugins to interact with resources. + * User information is fetched from thread context by security plugin. + * In clusters, where security plugin is disabled these requests will be pass-through via a No-op implementation. + * There are 3 scope of sharing for a resource: Private, Restricted, Public. To learn more visit {@linkplain this design proposal.} + * If security plugin is disabled, all resources will be considered public by default. + * TODO: add documentation around "how to use" + * + * + * + * @opensearch.experimental + */ +public interface ResourcePlugin { + + /** + * Returns all accessible resources for current user. + * + * @return list of {@link Resource} items accessible by current user. + */ + List listAccessibleResources(); + + /** + * Checks whether current user has permission to given resource. + * + * + * @param resource the resource on which access is to be checked + * @return true if current user has access, false otherwise + */ + boolean hasPermission(Resource resource); + + /** + * Adds an entity to the share-with. Resource needs to be in restricted mode. + * @param type One of the {@link ShareWith} types + * @param entities List of names with whom to share this resource with + * @return a message whether sharing was successful. + */ + String shareWith(ShareWith type, List entities); + + /** + * Revokes given permission to a resource + * + * @param resourceId if of the resource to be updated + * @param systemIndexName index where this resource is defined + * @param revokeAccess a map that contains entries of entities whose access should be revoked + * @return true if revoke was successful, false if there was a failure + */ + boolean revoke(String resourceId, String systemIndexName, Map> revokeAccess); + + /** + * Deletes an entry from .resource_sharing index + * @param resource The resource to be removed from the index + * @return true if resource record was deleted, false otherwise + */ + boolean deleteResourceSharingRecord(Resource resource); + + // TODO: Check whether methods for bulk updates are required +} From 66a849c3e548dc19e00471380956ad49a67ce14d Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Tue, 27 Aug 2024 14:34:50 -0400 Subject: [PATCH 002/108] Adds a No-op implementation of ResourcePlugin Signed-off-by: Darshit Chanpura --- .../plugins/NoOpResourcePlugin.java | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 server/src/main/java/org/opensearch/plugins/NoOpResourcePlugin.java diff --git a/server/src/main/java/org/opensearch/plugins/NoOpResourcePlugin.java b/server/src/main/java/org/opensearch/plugins/NoOpResourcePlugin.java new file mode 100644 index 0000000000000..c0878a3855187 --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/NoOpResourcePlugin.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.accesscontrol.resources.Resource; +import org.opensearch.accesscontrol.resources.ShareWith; + +import java.util.List; +import java.util.Map; + +/** + * This plugin class defines a no-op implementation of Resource Plugin. + * + * @opensearch.experimental + */ +public class NoOpResourcePlugin implements ResourcePlugin { + + /** + * Returns an empty list since security plugin is not defined. + * This method alone doesn't determine permissions. + * + * @return empty list + */ + @Override + public List listAccessibleResources() { + // returns an empty list since security plugin is disabled + return List.of(); + } + + /** + * Returns true since no authorization is required. + * + * @param resource the resource on which access is to be checked + * @return true + */ + public boolean hasPermission(Resource resource) { + return true; + } + + /** + * Adds an entity to the share-with. Resource needs to be in restricted mode. + * + * @param type One of the {@link ShareWith} types + * @param entities List of names with whom to share this resource with + * @return a message whether sharing was successful. + */ + public String shareWith(ShareWith type, List entities) { + + return "Unable to share as security plugin is disabled in the cluster."; + } + + /** + * Revokes access to the resource + * + * @param resourceId if of the resource to be updated + * @param systemIndexName index where this resource is defined + * @param revokeAccess a map that contains entries of entities whose access should be revoked + * @return false since no resource-sharing information is required as security plugin is disabled + */ + @Override + public boolean revoke(String resourceId, String systemIndexName, Map> revokeAccess) { + return false; + } + + /** + * Delete a resource sharing record + * @param resource The resource to be removed from the index + * @return false since security plugin is disabled + */ + @Override + public boolean deleteResourceSharingRecord(Resource resource) { + return false; + } + +} From d7169e4847e44724b659f56a587fac317f245b51 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 30 Aug 2024 12:19:25 -0400 Subject: [PATCH 003/108] Adds a way to configure security plugin for resource access-control Signed-off-by: Darshit Chanpura --- .../resources/ResourceService.java | 49 +++++++++++++++++++ .../main/java/org/opensearch/node/Node.java | 8 +++ .../plugins/ResourceAccessControlPlugin.java | 17 +++++++ 3 files changed, 74 insertions(+) create mode 100644 server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java create mode 100644 server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java new file mode 100644 index 0000000000000..b95d693b07b72 --- /dev/null +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.accesscontrol.resources; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; +import org.opensearch.plugins.NoOpResourcePlugin; +import org.opensearch.plugins.ResourceAccessControlPlugin; +import org.opensearch.plugins.ResourcePlugin; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Resource access control for OpenSearch + * + * @opensearch.experimental + * */ +public class ResourceService { + private static final Logger log = LogManager.getLogger(ResourceService.class); + + private final ResourcePlugin resourcePlugin; + + public ResourceService(final List resourcePlugins) { + if (resourcePlugins.size() == 0) { + log.debug("Security plugin disabled: Using NoopResourcePlugin"); + resourcePlugin = new NoOpResourcePlugin(); + } else if (resourcePlugins.size() == 1) { + log.debug("Security plugin enabled: Using OpenSearchSecurityPlugin"); + resourcePlugin = resourcePlugins.get(0); + } else { + throw new OpenSearchException( + "Multiple resource access control plugins are not supported, found: " + + resourcePlugins.stream().map(Object::getClass).map(Class::getName).collect(Collectors.joining(",")) + ); + } + } + + /** + * Gets the current ResourcePlugin to perform authorization + */ + public ResourcePlugin getResourceAccessControlPlugin() { + return resourcePlugin; + } +} diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index ea656af6110e5..01373adbc28df 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -41,6 +41,7 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.OpenSearchTimeoutException; import org.opensearch.Version; +import org.opensearch.accesscontrol.resources.ResourceService; import org.opensearch.action.ActionModule; import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.ActionType; @@ -212,6 +213,7 @@ import org.opensearch.plugins.Plugin; import org.opensearch.plugins.PluginsService; import org.opensearch.plugins.RepositoryPlugin; +import org.opensearch.plugins.ResourceAccessControlPlugin; import org.opensearch.plugins.ScriptPlugin; import org.opensearch.plugins.SearchPipelinePlugin; import org.opensearch.plugins.SearchPlugin; @@ -1058,6 +1060,11 @@ protected Node( ); modules.add(actionModule); + final List resourceAccessControlPlugins = pluginsService.filterPlugins( + ResourceAccessControlPlugin.class + ); + ResourceService resourceService = new ResourceService(resourceAccessControlPlugins); + final RestController restController = actionModule.getRestController(); final NodeResourceUsageTracker nodeResourceUsageTracker = new NodeResourceUsageTracker( @@ -1454,6 +1461,7 @@ protected Node( b.bind(ResourceUsageCollectorService.class).toInstance(resourceUsageCollectorService); b.bind(SystemIndices.class).toInstance(systemIndices); b.bind(IdentityService.class).toInstance(identityService); + b.bind(ResourceService.class).toInstance(resourceService); b.bind(Tracer.class).toInstance(tracer); b.bind(SearchRequestStats.class).toInstance(searchRequestStats); b.bind(SearchRequestSlowLog.class).toInstance(searchRequestSlowLog); diff --git a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java new file mode 100644 index 0000000000000..05fd6eb9aeddd --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java @@ -0,0 +1,17 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +/** + * Class to determine presence of security plugin in the cluster. + * If yes, security plugin will be used for resource access authorization + * + * @opensearch.experimental + */ +public interface ResourceAccessControlPlugin extends ResourcePlugin {} From 58ae851b40ff69ae4fbcd5a2d6bb85045af1d50c Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 30 Aug 2024 12:40:44 -0400 Subject: [PATCH 004/108] Fixes compilation errors and changes debug log-level to info for ResourceService Signed-off-by: Darshit Chanpura --- .../opensearch/accesscontrol/resources/ResourceService.java | 4 ++-- .../src/main/java/org/opensearch/plugins/ResourcePlugin.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java index b95d693b07b72..cd4dbdb91a0df 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java @@ -27,10 +27,10 @@ public class ResourceService { public ResourceService(final List resourcePlugins) { if (resourcePlugins.size() == 0) { - log.debug("Security plugin disabled: Using NoopResourcePlugin"); + log.info("Security plugin disabled: Using NoOpResourcePlugin"); resourcePlugin = new NoOpResourcePlugin(); } else if (resourcePlugins.size() == 1) { - log.debug("Security plugin enabled: Using OpenSearchSecurityPlugin"); + log.info("Security plugin enabled: Using OpenSearchSecurityPlugin"); resourcePlugin = resourcePlugins.get(0); } else { throw new OpenSearchException( diff --git a/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java b/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java index 3a357fd16c4a9..bd554c02d2dbb 100644 --- a/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java @@ -18,7 +18,7 @@ * This plugin class defines usage mechanisms for plugins to interact with resources. * User information is fetched from thread context by security plugin. * In clusters, where security plugin is disabled these requests will be pass-through via a No-op implementation. - * There are 3 scope of sharing for a resource: Private, Restricted, Public. To learn more visit {@linkplain this design proposal.} + * There are 3 scope of sharing for a resource: Private, Restricted, Public. To learn more visit ... * If security plugin is disabled, all resources will be considered public by default. * TODO: add documentation around "how to use" * From fd002439af83fa9668e5df97257ec06d12f1857e Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 30 Aug 2024 12:41:46 -0400 Subject: [PATCH 005/108] Replace plugin count check with isEmpty Signed-off-by: Darshit Chanpura --- .../org/opensearch/accesscontrol/resources/ResourceService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java index cd4dbdb91a0df..0786df5b03885 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java @@ -26,7 +26,7 @@ public class ResourceService { private final ResourcePlugin resourcePlugin; public ResourceService(final List resourcePlugins) { - if (resourcePlugins.size() == 0) { + if (resourcePlugins.isEmpty()) { log.info("Security plugin disabled: Using NoOpResourcePlugin"); resourcePlugin = new NoOpResourcePlugin(); } else if (resourcePlugins.size() == 1) { From ef8a0b70e4e787db3b2faeb45cd56637b594c59a Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 30 Aug 2024 12:53:49 -0400 Subject: [PATCH 006/108] Adds package-info Signed-off-by: Darshit Chanpura --- .../accesscontrol/resources/package-info.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java b/server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java new file mode 100644 index 0000000000000..65fe5c5f05713 --- /dev/null +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Actions that OpenSearch can take either on the data stored on disk or on other nodes. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/** + * This package defines all classes required for Resource Sharing and Access Control + */ +package org.opensearch.accesscontrol.resources; From e98cb61b3711392ef41bc9a737f4e791c93806dc Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 30 Aug 2024 13:36:46 -0400 Subject: [PATCH 007/108] Renames a bunch of files Signed-off-by: Darshit Chanpura --- .../resources/ResourceService.java | 32 ++++++---- .../main/java/org/opensearch/node/Node.java | 4 +- ...a => NoOpResourceAccessControlPlugin.java} | 2 +- .../plugins/ResourceAccessControlPlugin.java | 58 +++++++++++++++++- .../opensearch/plugins/ResourcePlugin.java | 61 +------------------ 5 files changed, 83 insertions(+), 74 deletions(-) rename server/src/main/java/org/opensearch/plugins/{NoOpResourcePlugin.java => NoOpResourceAccessControlPlugin.java} (96%) diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java index 0786df5b03885..4c403fed834ff 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java @@ -8,7 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; -import org.opensearch.plugins.NoOpResourcePlugin; +import org.opensearch.plugins.NoOpResourceAccessControlPlugin; import org.opensearch.plugins.ResourceAccessControlPlugin; import org.opensearch.plugins.ResourcePlugin; @@ -23,19 +23,22 @@ public class ResourceService { private static final Logger log = LogManager.getLogger(ResourceService.class); - private final ResourcePlugin resourcePlugin; + private final ResourceAccessControlPlugin resourceACPlugin; + private final List resourcePlugins; - public ResourceService(final List resourcePlugins) { - if (resourcePlugins.isEmpty()) { - log.info("Security plugin disabled: Using NoOpResourcePlugin"); - resourcePlugin = new NoOpResourcePlugin(); - } else if (resourcePlugins.size() == 1) { + public ResourceService(final List resourceACPlugins, List resourcePlugins) { + this.resourcePlugins = resourcePlugins; + + if (resourceACPlugins.isEmpty()) { + log.info("Security plugin disabled: Using NoOpResourceAccessControlPlugin"); + resourceACPlugin = new NoOpResourceAccessControlPlugin(); + } else if (resourceACPlugins.size() == 1) { log.info("Security plugin enabled: Using OpenSearchSecurityPlugin"); - resourcePlugin = resourcePlugins.get(0); + resourceACPlugin = resourceACPlugins.get(0); } else { throw new OpenSearchException( "Multiple resource access control plugins are not supported, found: " - + resourcePlugins.stream().map(Object::getClass).map(Class::getName).collect(Collectors.joining(",")) + + resourceACPlugins.stream().map(Object::getClass).map(Class::getName).collect(Collectors.joining(",")) ); } } @@ -43,7 +46,14 @@ public ResourceService(final List resourcePlugins) /** * Gets the current ResourcePlugin to perform authorization */ - public ResourcePlugin getResourceAccessControlPlugin() { - return resourcePlugin; + public ResourceAccessControlPlugin getResourceAccessControlPlugin() { + return resourceACPlugin; + } + + /** + * List active plugins that define resources + */ + public List listResourcePlugins() { + return resourcePlugins; } } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 01373adbc28df..792a1b9af42f2 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -214,6 +214,7 @@ import org.opensearch.plugins.PluginsService; import org.opensearch.plugins.RepositoryPlugin; import org.opensearch.plugins.ResourceAccessControlPlugin; +import org.opensearch.plugins.ResourcePlugin; import org.opensearch.plugins.ScriptPlugin; import org.opensearch.plugins.SearchPipelinePlugin; import org.opensearch.plugins.SearchPlugin; @@ -1063,7 +1064,8 @@ protected Node( final List resourceAccessControlPlugins = pluginsService.filterPlugins( ResourceAccessControlPlugin.class ); - ResourceService resourceService = new ResourceService(resourceAccessControlPlugins); + final List resourcePlugins = pluginsService.filterPlugins(ResourcePlugin.class); + ResourceService resourceService = new ResourceService(resourceAccessControlPlugins, resourcePlugins); final RestController restController = actionModule.getRestController(); diff --git a/server/src/main/java/org/opensearch/plugins/NoOpResourcePlugin.java b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java similarity index 96% rename from server/src/main/java/org/opensearch/plugins/NoOpResourcePlugin.java rename to server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java index c0878a3855187..2d263b8c359ab 100644 --- a/server/src/main/java/org/opensearch/plugins/NoOpResourcePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java @@ -19,7 +19,7 @@ * * @opensearch.experimental */ -public class NoOpResourcePlugin implements ResourcePlugin { +public class NoOpResourceAccessControlPlugin implements ResourceAccessControlPlugin { /** * Returns an empty list since security plugin is not defined. diff --git a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java index 05fd6eb9aeddd..02fbcb6a7563a 100644 --- a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java @@ -8,10 +8,64 @@ package org.opensearch.plugins; +import org.opensearch.accesscontrol.resources.Resource; +import org.opensearch.accesscontrol.resources.ShareWith; + +import java.util.List; +import java.util.Map; + /** - * Class to determine presence of security plugin in the cluster. + * This interface determines presence of security plugin in the cluster. * If yes, security plugin will be used for resource access authorization + * User information is fetched from thread context by security plugin. + * In clusters, where security plugin is disabled these requests will be pass-through via a No-op implementation. + * There are 3 scope of sharing for a resource: Private, Restricted, Public. To learn more visit ... + * If security plugin is disabled, all resources will be considered public by default. + * TODO: add documentation around "how to use" * * @opensearch.experimental */ -public interface ResourceAccessControlPlugin extends ResourcePlugin {} +public interface ResourceAccessControlPlugin { + /** + * Returns all accessible resources for current user. + * + * @return list of {@link Resource} items accessible by current user. + */ + List listAccessibleResources(); + + /** + * Checks whether current user has permission to given resource. + * + * + * @param resource the resource on which access is to be checked + * @return true if current user has access, false otherwise + */ + boolean hasPermission(Resource resource); + + /** + * Adds an entity to the share-with. Resource needs to be in restricted mode. + * @param type One of the {@link ShareWith} types + * @param entities List of names with whom to share this resource with + * @return a message whether sharing was successful. + */ + String shareWith(ShareWith type, List entities); + + /** + * Revokes given permission to a resource + * + * @param resourceId if of the resource to be updated + * @param systemIndexName index where this resource is defined + * @param revokeAccess a map that contains entries of entities whose access should be revoked + * @return true if revoke was successful, false if there was a failure + */ + boolean revoke(String resourceId, String systemIndexName, Map> revokeAccess); + + /** + * Deletes an entry from .resource_sharing index + * @param resource The resource to be removed from the index + * @return true if resource record was deleted, false otherwise + */ + boolean deleteResourceSharingRecord(Resource resource); + + // TODO: Check whether methods for bulk updates are required +} diff --git a/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java b/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java index bd554c02d2dbb..e6e75d68f8626 100644 --- a/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java @@ -8,66 +8,9 @@ package org.opensearch.plugins; -import org.opensearch.accesscontrol.resources.Resource; -import org.opensearch.accesscontrol.resources.ShareWith; - -import java.util.List; -import java.util.Map; - /** - * This plugin class defines usage mechanisms for plugins to interact with resources. - * User information is fetched from thread context by security plugin. - * In clusters, where security plugin is disabled these requests will be pass-through via a No-op implementation. - * There are 3 scope of sharing for a resource: Private, Restricted, Public. To learn more visit ... - * If security plugin is disabled, all resources will be considered public by default. - * TODO: add documentation around "how to use" - * - * + * This interface should be implemented by all the plugins that define one or more resources. * * @opensearch.experimental */ -public interface ResourcePlugin { - - /** - * Returns all accessible resources for current user. - * - * @return list of {@link Resource} items accessible by current user. - */ - List listAccessibleResources(); - - /** - * Checks whether current user has permission to given resource. - * - * - * @param resource the resource on which access is to be checked - * @return true if current user has access, false otherwise - */ - boolean hasPermission(Resource resource); - - /** - * Adds an entity to the share-with. Resource needs to be in restricted mode. - * @param type One of the {@link ShareWith} types - * @param entities List of names with whom to share this resource with - * @return a message whether sharing was successful. - */ - String shareWith(ShareWith type, List entities); - - /** - * Revokes given permission to a resource - * - * @param resourceId if of the resource to be updated - * @param systemIndexName index where this resource is defined - * @param revokeAccess a map that contains entries of entities whose access should be revoked - * @return true if revoke was successful, false if there was a failure - */ - boolean revoke(String resourceId, String systemIndexName, Map> revokeAccess); - - /** - * Deletes an entry from .resource_sharing index - * @param resource The resource to be removed from the index - * @return true if resource record was deleted, false otherwise - */ - boolean deleteResourceSharingRecord(Resource resource); - - // TODO: Check whether methods for bulk updates are required -} +public interface ResourcePlugin {} From 96f09b0a88e43c2901210acb8336a81b47f67f6f Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 30 Aug 2024 14:55:14 -0400 Subject: [PATCH 008/108] Changes method signatures to be inline with their usage Signed-off-by: Darshit Chanpura --- .../accesscontrol/resources/EntityType.java | 23 ++++ .../accesscontrol/resources/Resource.java | 101 ------------------ .../resources/ResourceSharing.java | 86 +++++++++++---- .../accesscontrol/resources/ShareWith.java | 48 ++++++++- .../NoOpResourceAccessControlPlugin.java | 45 +++++--- .../plugins/ResourceAccessControlPlugin.java | 38 ++++--- 6 files changed, 181 insertions(+), 160 deletions(-) create mode 100644 server/src/main/java/org/opensearch/accesscontrol/resources/EntityType.java delete mode 100644 server/src/main/java/org/opensearch/accesscontrol/resources/Resource.java diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/EntityType.java b/server/src/main/java/org/opensearch/accesscontrol/resources/EntityType.java new file mode 100644 index 0000000000000..435172f4e5097 --- /dev/null +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/EntityType.java @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.accesscontrol.resources; + +/** + * This enum contains the type of entities a resource can be shared with. + * + * @opensearch.experimental + */ +public enum EntityType { + + USERS, + + ROLES, + + BACKEND_ROLES, +} diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/Resource.java b/server/src/main/java/org/opensearch/accesscontrol/resources/Resource.java deleted file mode 100644 index 4212cb630cced..0000000000000 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/Resource.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.accesscontrol.resources; - -import java.util.Objects; - -/** - * A document in .resource_sharing index. - * Holds information about the resource (obtained from defining plugin's meta-data), - * the index which defines the resources, the creator of the resource, - * and the information on whom this resource is shared with. - * - * @opensearch.experimental - */ -public class Resource { - - private String sourceIdx; - - private String resourceId; - - private CreatedBy createdBy; - - private ResourceSharing sharedWith; - - public Resource(String sourceIdx, String resourceId, CreatedBy createdBy, ResourceSharing sharedWith) { - this.sourceIdx = sourceIdx; - this.resourceId = resourceId; - this.createdBy = createdBy; - this.sharedWith = sharedWith; - } - - public String getSourceIdx() { - return sourceIdx; - } - - public void setSourceIdx(String sourceIdx) { - this.sourceIdx = sourceIdx; - } - - public String getResourceId() { - return resourceId; - } - - public void setResourceId(String resourceId) { - this.resourceId = resourceId; - } - - public CreatedBy getCreatedBy() { - return createdBy; - } - - public void setCreatedBy(CreatedBy createdBy) { - this.createdBy = createdBy; - } - - public ResourceSharing getSharedWith() { - return sharedWith; - } - - public void setSharedWith(ResourceSharing sharedWith) { - this.sharedWith = sharedWith; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Resource resource = (Resource) o; - return Objects.equals(getSourceIdx(), resource.getSourceIdx()) - && Objects.equals(getResourceId(), resource.getResourceId()) - && Objects.equals(getCreatedBy(), resource.getCreatedBy()) - && Objects.equals(getSharedWith(), resource.getSharedWith()); - } - - @Override - public int hashCode() { - return Objects.hash(getSourceIdx(), getResourceId(), getCreatedBy(), getSharedWith()); - } - - @Override - public String toString() { - return "Resource {" - + "sourceIdx='" - + sourceIdx - + '\'' - + ", resourceId='" - + resourceId - + '\'' - + ", createdBy=" - + createdBy - + ", sharedWith=" - + sharedWith - + '}'; - } -} diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java index 331ccb5de898c..7c866e0657a02 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java @@ -8,54 +8,94 @@ package org.opensearch.accesscontrol.resources; -import java.util.List; +import java.util.Objects; /** - * This class contains information about whom a resource is shared with. - * It could be a user-name, a role or a backend_role. + * A document in .resource_sharing index. + * Holds information about the resource (obtained from defining plugin's meta-data), + * the index which defines the resources, the creator of the resource, + * and the information on whom this resource is shared with. * * @opensearch.experimental */ public class ResourceSharing { - private List users; + private String sourceIdx; - private List roles; + private String resourceId; - private List backendRoles; + private CreatedBy createdBy; - public ResourceSharing(List users, List backendRoles, List roles) { - this.users = users; - this.backendRoles = backendRoles; - this.roles = roles; + private ShareWith sharedWith; + + public ResourceSharing(String sourceIdx, String resourceId, CreatedBy createdBy, ShareWith sharedWith) { + this.sourceIdx = sourceIdx; + this.resourceId = resourceId; + this.createdBy = createdBy; + this.sharedWith = sharedWith; + } + + public String getSourceIdx() { + return sourceIdx; } - public List getUsers() { - return users; + public void setSourceIdx(String sourceIdx) { + this.sourceIdx = sourceIdx; } - public void setUsers(List users) { - this.users = users; + public String getResourceId() { + return resourceId; } - public List getRoles() { - return roles; + public void setResourceId(String resourceId) { + this.resourceId = resourceId; } - public void setRoles(List roles) { - this.roles = roles; + public CreatedBy getCreatedBy() { + return createdBy; } - public List getBackendRoles() { - return backendRoles; + public void setCreatedBy(CreatedBy createdBy) { + this.createdBy = createdBy; } - public void setBackendRoles(List backendRoles) { - this.backendRoles = backendRoles; + public ShareWith getSharedWith() { + return sharedWith; + } + + public void setSharedWith(ShareWith sharedWith) { + this.sharedWith = sharedWith; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ResourceSharing resourceSharing = (ResourceSharing) o; + return Objects.equals(getSourceIdx(), resourceSharing.getSourceIdx()) + && Objects.equals(getResourceId(), resourceSharing.getResourceId()) + && Objects.equals(getCreatedBy(), resourceSharing.getCreatedBy()) + && Objects.equals(getSharedWith(), resourceSharing.getSharedWith()); + } + + @Override + public int hashCode() { + return Objects.hash(getSourceIdx(), getResourceId(), getCreatedBy(), getSharedWith()); } @Override public String toString() { - return "ResourceSharing {" + "users=" + users + ", roles=" + roles + ", backendRoles=" + backendRoles + '}'; + return "Resource {" + + "sourceIdx='" + + sourceIdx + + '\'' + + ", resourceId='" + + resourceId + + '\'' + + ", createdBy=" + + createdBy + + ", sharedWith=" + + sharedWith + + '}'; } } diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java index 702081ebd9ec9..980a4cd55b35e 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java @@ -8,16 +8,54 @@ package org.opensearch.accesscontrol.resources; +import java.util.List; + /** - * This enum contains the type of entities a resource can be shared with. + * This class contains information about whom a resource is shared with. + * It could be a user-name, a role or a backend_role. * * @opensearch.experimental */ -public enum ShareWith { +public class ShareWith { + + private List users; + + private List roles; + + private List backendRoles; + + public ShareWith(List users, List backendRoles, List roles) { + this.users = users; + this.backendRoles = backendRoles; + this.roles = roles; + } + + public List getUsers() { + return users; + } + + public void setUsers(List users) { + this.users = users; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } - USERS, + public List getBackendRoles() { + return backendRoles; + } - ROLES, + public void setBackendRoles(List backendRoles) { + this.backendRoles = backendRoles; + } - BACKEND_ROLES, + @Override + public String toString() { + return "ShareWith {" + "users=" + users + ", roles=" + roles + ", backendRoles=" + backendRoles + '}'; + } } diff --git a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java index 2d263b8c359ab..743d3e248e1da 100644 --- a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java @@ -8,8 +8,8 @@ package org.opensearch.plugins; -import org.opensearch.accesscontrol.resources.Resource; -import org.opensearch.accesscontrol.resources.ShareWith; +import org.opensearch.accesscontrol.resources.EntityType; +import org.opensearch.accesscontrol.resources.ResourceSharing; import java.util.List; import java.util.Map; @@ -21,6 +21,15 @@ */ public class NoOpResourceAccessControlPlugin implements ResourceAccessControlPlugin { + /** + * + * @return an empty map of resource names accessible by this user. + */ + @Override + public Map> listAccessibleResources() { + return Map.of(); + } + /** * Returns an empty list since security plugin is not defined. * This method alone doesn't determine permissions. @@ -28,7 +37,7 @@ public class NoOpResourceAccessControlPlugin implements ResourceAccessControlPlu * @return empty list */ @Override - public List listAccessibleResources() { + public List listAccessibleResourcesForPlugin(String systemIndexName) { // returns an empty list since security plugin is disabled return List.of(); } @@ -36,23 +45,26 @@ public List listAccessibleResources() { /** * Returns true since no authorization is required. * - * @param resource the resource on which access is to be checked + * @param resourceId the resource on which access is to be checked + * @param systemIndexName where the resource exists * @return true */ - public boolean hasPermission(Resource resource) { + @Override + public boolean hasPermission(String resourceId, String systemIndexName) { return true; } /** * Adds an entity to the share-with. Resource needs to be in restricted mode. * - * @param type One of the {@link ShareWith} types - * @param entities List of names with whom to share this resource with - * @return a message whether sharing was successful. + * @param resourceId if of the resource to be updated + * @param systemIndexName index where this resource is defined + * @param revokeAccess a map that contains entries of entities with whom this resource should be shared with + * @return null since security plugin is disabled in the cluster */ - public String shareWith(ShareWith type, List entities) { - - return "Unable to share as security plugin is disabled in the cluster."; + @Override + public ResourceSharing shareWith(String resourceId, String systemIndexName, Map> revokeAccess) { + return null; } /** @@ -61,20 +73,21 @@ public String shareWith(ShareWith type, List entities) { * @param resourceId if of the resource to be updated * @param systemIndexName index where this resource is defined * @param revokeAccess a map that contains entries of entities whose access should be revoked - * @return false since no resource-sharing information is required as security plugin is disabled + * @return null since security plugin is disabled in the cluster */ @Override - public boolean revoke(String resourceId, String systemIndexName, Map> revokeAccess) { - return false; + public ResourceSharing revokeAccess(String resourceId, String systemIndexName, Map> revokeAccess) { + return null; } /** * Delete a resource sharing record - * @param resource The resource to be removed from the index + * @param resourceId if of the resource to be updated + * @param systemIndexName index where this resource is defined * @return false since security plugin is disabled */ @Override - public boolean deleteResourceSharingRecord(Resource resource) { + public boolean deleteResourceSharingRecord(String resourceId, String systemIndexName) { return false; } diff --git a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java index 02fbcb6a7563a..e03f8e11aeccc 100644 --- a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java @@ -8,8 +8,8 @@ package org.opensearch.plugins; -import org.opensearch.accesscontrol.resources.Resource; -import org.opensearch.accesscontrol.resources.ShareWith; +import org.opensearch.accesscontrol.resources.EntityType; +import org.opensearch.accesscontrol.resources.ResourceSharing; import java.util.List; import java.util.Map; @@ -28,27 +28,34 @@ public interface ResourceAccessControlPlugin { /** * Returns all accessible resources for current user. + * @return a map of resources accessible by user separated by system index names + */ + Map> listAccessibleResources(); + + /** + * Returns all accessible resources for current user for a given system . * - * @return list of {@link Resource} items accessible by current user. + * @return list of {@link ResourceSharing} items accessible by current user. */ - List listAccessibleResources(); + List listAccessibleResourcesForPlugin(String systemIndex); /** * Checks whether current user has permission to given resource. * - * - * @param resource the resource on which access is to be checked + * @param resourceId the resource on which access is to be checked + * @param systemIndexName where the resource exists * @return true if current user has access, false otherwise */ - boolean hasPermission(Resource resource); + boolean hasPermission(String resourceId, String systemIndexName); /** * Adds an entity to the share-with. Resource needs to be in restricted mode. - * @param type One of the {@link ShareWith} types - * @param entities List of names with whom to share this resource with - * @return a message whether sharing was successful. + * @param resourceId if of the resource to be updated + * @param systemIndexName index where this resource is defined + * @param entities a map that contains entries of entities with whom the resource should be shared with + * @return updated resource sharing record */ - String shareWith(ShareWith type, List entities); + ResourceSharing shareWith(String resourceId, String systemIndexName, Map> entities); /** * Revokes given permission to a resource @@ -56,16 +63,17 @@ public interface ResourceAccessControlPlugin { * @param resourceId if of the resource to be updated * @param systemIndexName index where this resource is defined * @param revokeAccess a map that contains entries of entities whose access should be revoked - * @return true if revoke was successful, false if there was a failure + * @return the updated ResourceSharing record */ - boolean revoke(String resourceId, String systemIndexName, Map> revokeAccess); + ResourceSharing revokeAccess(String resourceId, String systemIndexName, Map> revokeAccess); /** * Deletes an entry from .resource_sharing index - * @param resource The resource to be removed from the index + * @param resourceId if of the resource to be updated + * @param systemIndexName index where this resource is defined * @return true if resource record was deleted, false otherwise */ - boolean deleteResourceSharingRecord(Resource resource); + boolean deleteResourceSharingRecord(String resourceId, String systemIndexName); // TODO: Check whether methods for bulk updates are required } From c86dfc98ccd868feb2ed09dd5470bae18d825bda Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 30 Aug 2024 15:03:12 -0400 Subject: [PATCH 009/108] Adds new method for deleting by entity Signed-off-by: Darshit Chanpura --- .../plugins/NoOpResourceAccessControlPlugin.java | 16 +++++++++------- .../plugins/ResourceAccessControlPlugin.java | 7 +++++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java index 743d3e248e1da..af6e79e34f84c 100644 --- a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java @@ -43,8 +43,6 @@ public List listAccessibleResourcesForPlugin(String systemIndexName) { } /** - * Returns true since no authorization is required. - * * @param resourceId the resource on which access is to be checked * @param systemIndexName where the resource exists * @return true @@ -55,8 +53,6 @@ public boolean hasPermission(String resourceId, String systemIndexName) { } /** - * Adds an entity to the share-with. Resource needs to be in restricted mode. - * * @param resourceId if of the resource to be updated * @param systemIndexName index where this resource is defined * @param revokeAccess a map that contains entries of entities with whom this resource should be shared with @@ -68,8 +64,6 @@ public ResourceSharing shareWith(String resourceId, String systemIndexName, Map< } /** - * Revokes access to the resource - * * @param resourceId if of the resource to be updated * @param systemIndexName index where this resource is defined * @param revokeAccess a map that contains entries of entities whose access should be revoked @@ -81,7 +75,6 @@ public ResourceSharing revokeAccess(String resourceId, String systemIndexName, M } /** - * Delete a resource sharing record * @param resourceId if of the resource to be updated * @param systemIndexName index where this resource is defined * @return false since security plugin is disabled @@ -91,4 +84,13 @@ public boolean deleteResourceSharingRecord(String resourceId, String systemIndex return false; } + /** + * @param entity whose resource sharing entries are to be deleted + * @return false since security plugin is disabled + */ + @Override + public boolean deleteAllResourceSharingRecordsFor(String entity) { + return false; + } + } diff --git a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java index e03f8e11aeccc..6bcd3150b59db 100644 --- a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java @@ -75,5 +75,12 @@ public interface ResourceAccessControlPlugin { */ boolean deleteResourceSharingRecord(String resourceId, String systemIndexName); + /** + * Deletes all entries from .resource_sharing index where requested entity is the creator of the resource + * @param entity whose resource sharing records are to be deleted + * @return true if resource record was deleted, false otherwise + */ + boolean deleteAllResourceSharingRecordsFor(String entity); + // TODO: Check whether methods for bulk updates are required } From 7c6ec2a670ea7fa27aa8be19b61ab43ff9f5d3c2 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Tue, 3 Sep 2024 11:09:03 -0400 Subject: [PATCH 010/108] Adds abstract method definitions for ResourcePlugin interface Signed-off-by: Darshit Chanpura --- .../org/opensearch/plugins/ResourcePlugin.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java b/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java index e6e75d68f8626..4fbbbf35fc367 100644 --- a/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java @@ -13,4 +13,17 @@ * * @opensearch.experimental */ -public interface ResourcePlugin {} +public interface ResourcePlugin { + + /** + * Type of the resource + * @return a string containing the type of the resource + */ + String getResourceType(); + + /** + * The index where resource meta-data is stored + * @return the name of the parent index where resource meta-data is stored + */ + String getResourceIndex(); +} From f95a67f77d12b0cd4815dcd23bc25998c7a50618 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 6 Sep 2024 12:29:54 -0400 Subject: [PATCH 011/108] Adds toXContent implementations Signed-off-by: Darshit Chanpura --- .../accesscontrol/resources/CreatedBy.java | 12 ++++++- .../resources/ResourceSharing.java | 36 +++++++++++++------ .../accesscontrol/resources/ShareWith.java | 15 ++++++-- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java b/server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java index 01c61e82ed5f4..d5c521e6c20f3 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java @@ -8,13 +8,18 @@ package org.opensearch.accesscontrol.resources; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + /** * This class contains information on the creator of a resource. * Creator can either be a user or a backend_role. * * @opensearch.experimental */ -public class CreatedBy { +public class CreatedBy implements ToXContentFragment { private String user; @@ -45,4 +50,9 @@ public void setUser(String user) { public String toString() { return "CreatedBy {" + "user='" + user + '\'' + ", backendRole='" + backendRole + '\'' + '}'; } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject().field("user", user).field("backend_role", backendRole).endObject(); + } } diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java index 7c866e0657a02..bb5cdfb78a41e 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java @@ -8,6 +8,10 @@ package org.opensearch.accesscontrol.resources; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; import java.util.Objects; /** @@ -18,7 +22,7 @@ * * @opensearch.experimental */ -public class ResourceSharing { +public class ResourceSharing implements ToXContentFragment { private String sourceIdx; @@ -26,13 +30,13 @@ public class ResourceSharing { private CreatedBy createdBy; - private ShareWith sharedWith; + private ShareWith shareWith; - public ResourceSharing(String sourceIdx, String resourceId, CreatedBy createdBy, ShareWith sharedWith) { + public ResourceSharing(String sourceIdx, String resourceId, CreatedBy createdBy, ShareWith shareWith) { this.sourceIdx = sourceIdx; this.resourceId = resourceId; this.createdBy = createdBy; - this.sharedWith = sharedWith; + this.shareWith = shareWith; } public String getSourceIdx() { @@ -59,12 +63,12 @@ public void setCreatedBy(CreatedBy createdBy) { this.createdBy = createdBy; } - public ShareWith getSharedWith() { - return sharedWith; + public ShareWith getShareWith() { + return shareWith; } - public void setSharedWith(ShareWith sharedWith) { - this.sharedWith = sharedWith; + public void setShareWith(ShareWith shareWith) { + this.shareWith = shareWith; } @Override @@ -75,12 +79,12 @@ public boolean equals(Object o) { return Objects.equals(getSourceIdx(), resourceSharing.getSourceIdx()) && Objects.equals(getResourceId(), resourceSharing.getResourceId()) && Objects.equals(getCreatedBy(), resourceSharing.getCreatedBy()) - && Objects.equals(getSharedWith(), resourceSharing.getSharedWith()); + && Objects.equals(getShareWith(), resourceSharing.getShareWith()); } @Override public int hashCode() { - return Objects.hash(getSourceIdx(), getResourceId(), getCreatedBy(), getSharedWith()); + return Objects.hash(getSourceIdx(), getResourceId(), getCreatedBy(), getShareWith()); } @Override @@ -95,7 +99,17 @@ public String toString() { + ", createdBy=" + createdBy + ", sharedWith=" - + sharedWith + + shareWith + '}'; } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field("source_idx", sourceIdx) + .field("resource_id", resourceId) + .field("created_by", createdBy) + .field("share_with", shareWith) + .endObject(); + } } diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java index 980a4cd55b35e..8bb3730bacd19 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java @@ -8,6 +8,10 @@ package org.opensearch.accesscontrol.resources; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; import java.util.List; /** @@ -16,7 +20,7 @@ * * @opensearch.experimental */ -public class ShareWith { +public class ShareWith implements ToXContentFragment { private List users; @@ -24,10 +28,10 @@ public class ShareWith { private List backendRoles; - public ShareWith(List users, List backendRoles, List roles) { + public ShareWith(List users, List roles, List backendRoles) { this.users = users; - this.backendRoles = backendRoles; this.roles = roles; + this.backendRoles = backendRoles; } public List getUsers() { @@ -58,4 +62,9 @@ public void setBackendRoles(List backendRoles) { public String toString() { return "ShareWith {" + "users=" + users + ", roles=" + roles + ", backendRoles=" + backendRoles + '}'; } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject().field("users", users).field("roles", roles).field("backend_roles", backendRoles).endObject(); + } } From 7e7cd0ad7bd0da264cc235cb8c379a5d9864ecbd Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Tue, 10 Sep 2024 12:50:32 -0400 Subject: [PATCH 012/108] Modifies some method names and comments Signed-off-by: Darshit Chanpura --- .../plugins/NoOpResourceAccessControlPlugin.java | 8 ++++---- .../plugins/ResourceAccessControlPlugin.java | 14 ++++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java index af6e79e34f84c..c3fae97380186 100644 --- a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java @@ -10,6 +10,7 @@ import org.opensearch.accesscontrol.resources.EntityType; import org.opensearch.accesscontrol.resources.ResourceSharing; +import org.opensearch.accesscontrol.resources.ShareWith; import java.util.List; import java.util.Map; @@ -55,11 +56,11 @@ public boolean hasPermission(String resourceId, String systemIndexName) { /** * @param resourceId if of the resource to be updated * @param systemIndexName index where this resource is defined - * @param revokeAccess a map that contains entries of entities with whom this resource should be shared with + * @param shareWith a map that contains entries of entities with whom this resource should be shared with * @return null since security plugin is disabled in the cluster */ @Override - public ResourceSharing shareWith(String resourceId, String systemIndexName, Map> revokeAccess) { + public ResourceSharing shareWith(String resourceId, String systemIndexName, ShareWith shareWith) { return null; } @@ -85,11 +86,10 @@ public boolean deleteResourceSharingRecord(String resourceId, String systemIndex } /** - * @param entity whose resource sharing entries are to be deleted * @return false since security plugin is disabled */ @Override - public boolean deleteAllResourceSharingRecordsFor(String entity) { + public boolean deleteAllResourceSharingRecordsForCurrentUser() { return false; } diff --git a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java index 6bcd3150b59db..f23fdefda0230 100644 --- a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java @@ -10,6 +10,7 @@ import org.opensearch.accesscontrol.resources.EntityType; import org.opensearch.accesscontrol.resources.ResourceSharing; +import org.opensearch.accesscontrol.resources.ShareWith; import java.util.List; import java.util.Map; @@ -50,12 +51,13 @@ public interface ResourceAccessControlPlugin { /** * Adds an entity to the share-with. Resource needs to be in restricted mode. - * @param resourceId if of the resource to be updated + * Creates a resource sharing record if one doesn't exist. + * @param resourceId id of the resource to be updated * @param systemIndexName index where this resource is defined - * @param entities a map that contains entries of entities with whom the resource should be shared with + * @param shareWith an object that contains entries of entities with whom the resource should be shared with * @return updated resource sharing record */ - ResourceSharing shareWith(String resourceId, String systemIndexName, Map> entities); + ResourceSharing shareWith(String resourceId, String systemIndexName, ShareWith shareWith); /** * Revokes given permission to a resource @@ -76,11 +78,11 @@ public interface ResourceAccessControlPlugin { boolean deleteResourceSharingRecord(String resourceId, String systemIndexName); /** - * Deletes all entries from .resource_sharing index where requested entity is the creator of the resource - * @param entity whose resource sharing records are to be deleted + * TODO check if this method is needed + * Deletes all entries from .resource_sharing index where current user is the creator of the resource * @return true if resource record was deleted, false otherwise */ - boolean deleteAllResourceSharingRecordsFor(String entity); + boolean deleteAllResourceSharingRecordsForCurrentUser(); // TODO: Check whether methods for bulk updates are required } From 4223fab40bedd0571324f76f8838ccea4a8b1b8a Mon Sep 17 00:00:00 2001 From: Sagar <99425694+sgup432@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:31:46 -0700 Subject: [PATCH 013/108] Removing task Id from map using remove() (#15918) Signed-off-by: Sagar Upadhyaya --- .../org/opensearch/tasks/TaskCancellationMonitoringService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java index 343d4571593a7..2040703d88c38 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java @@ -125,7 +125,7 @@ public void onTaskCompleted(Task task) { if (!TASKS_TO_TRACK.contains(task.getClass())) { return; } - this.cancelledTaskTracker.entrySet().removeIf(entry -> entry.getKey() == task.getId()); + this.cancelledTaskTracker.remove(task.getId()); } /** From 07029b2513f0cffbd788d95836dbc6cff0b4dbf4 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Fri, 13 Sep 2024 05:53:57 -0700 Subject: [PATCH 014/108] Add the capability to override indices.breaker.total.use_real_memory setting for test clusters (#15906) Signed-off-by: Navneet Verma --- .../org/opensearch/gradle/testclusters/OpenSearchNode.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java index 268de50340cbf..cd22560af9a96 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java @@ -116,7 +116,12 @@ public class OpenSearchNode implements TestClusterConfiguration { private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.MINUTES; private static final int ADDITIONAL_CONFIG_TIMEOUT = 15; private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS; - private static final List OVERRIDABLE_SETTINGS = Arrays.asList("path.repo", "discovery.seed_providers", "discovery.seed_hosts"); + private static final List OVERRIDABLE_SETTINGS = Arrays.asList( + "path.repo", + "discovery.seed_providers", + "discovery.seed_hosts", + "indices.breaker.total.use_real_memory" + ); private static final int TAIL_LOG_MESSAGES_COUNT = 40; private static final List MESSAGES_WE_DONT_CARE_ABOUT = Arrays.asList( From 260edc5f13debc04858da4e73dfc396eeb459ef9 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 13 Sep 2024 10:44:16 -0400 Subject: [PATCH 015/108] Fix flaky org.opensearch.rest.ReactorNetty4StreamingStressIT.testCloseClientStreamingRequest test case (#15859) Signed-off-by: Andriy Redko --- .../rest/ReactorNetty4StreamingStressIT.java | 56 +++++++------------ 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java index a978af1b11db4..a853ed56fad32 100644 --- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java +++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java @@ -8,7 +8,6 @@ package org.opensearch.rest; -import org.apache.hc.core5.http.ConnectionClosedException; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.StreamingRequest; @@ -16,24 +15,20 @@ import org.opensearch.test.rest.OpenSearchRestTestCase; import org.junit.After; +import java.io.IOException; import java.io.InterruptedIOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; import reactor.core.publisher.Flux; -import reactor.test.subscriber.TestSubscriber; +import reactor.test.StepVerifier; +import reactor.test.scheduler.VirtualTimeScheduler; -import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.collection.IsEmptyCollection.empty; public class ReactorNetty4StreamingStressIT extends OpenSearchRestTestCase { @After @@ -49,6 +44,8 @@ public void tearDown() throws Exception { } public void testCloseClientStreamingRequest() throws Exception { + final VirtualTimeScheduler scheduler = VirtualTimeScheduler.create(true); + final AtomicInteger id = new AtomicInteger(0); final Stream stream = Stream.generate( () -> "{ \"index\": { \"_index\": \"test-stress-streaming\", \"_id\": \"" @@ -57,39 +54,28 @@ public void testCloseClientStreamingRequest() throws Exception { + "{ \"name\": \"josh\" }\n" ); + final Duration delay = Duration.ofMillis(1); final StreamingRequest streamingRequest = new StreamingRequest<>( "POST", "/_bulk/stream", - Flux.fromStream(stream).delayElements(Duration.ofMillis(500)).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8))) + Flux.fromStream(stream).delayElements(delay, scheduler).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8))) ); streamingRequest.addParameter("refresh", "true"); final StreamingResponse streamingResponse = client().streamRequest(streamingRequest); - TestSubscriber subscriber = TestSubscriber.create(); - streamingResponse.getBody().subscribe(subscriber); - - final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - try { - // Await for subscriber to receive at least one chunk - assertBusy(() -> assertThat(subscriber.getReceivedOnNext(), not(empty()))); - - // Close client forceably - executor.schedule(() -> { - client().close(); - return null; - }, 2, TimeUnit.SECONDS); + scheduler.advanceTimeBy(delay); /* emit first element */ - // Await for subscriber to terminate - subscriber.block(Duration.ofSeconds(10)); - assertThat( - subscriber.expectTerminalError(), - anyOf(instanceOf(InterruptedIOException.class), instanceOf(ConnectionClosedException.class)) - ); - } finally { - executor.shutdown(); - if (executor.awaitTermination(1, TimeUnit.SECONDS) == false) { - executor.shutdownNow(); - } - } + StepVerifier.create(Flux.from(streamingResponse.getBody()).map(b -> new String(b.array(), StandardCharsets.UTF_8))) + .expectNextMatches(s -> s.contains("\"result\":\"created\"") && s.contains("\"_id\":\"1\"")) + .then(() -> { + try { + client().close(); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + }) + .then(() -> scheduler.advanceTimeBy(delay)) + .expectErrorMatches(t -> t instanceof InterruptedIOException) + .verify(); } } From 36c89bf4f975bd2750423d1308d2cd3408d8553f Mon Sep 17 00:00:00 2001 From: Finn Date: Fri, 13 Sep 2024 08:00:37 -0700 Subject: [PATCH 016/108] Update TESTING.md 'Building with extra plugins' documentation (#15893) * Update TESTING.md 'Building with extra plugins' documentation Signed-off-by: Finn Carroll * Typo Signed-off-by: Finn Carroll * Fix TOC Signed-off-by: Finn Carroll --------- Signed-off-by: Finn Carroll --- TESTING.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/TESTING.md b/TESTING.md index de7ab3eefe2f8..9127f005ced75 100644 --- a/TESTING.md +++ b/TESTING.md @@ -39,7 +39,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran - [Expect a specific segment topology](#expect-a-specific-segment-topology) - [Leave environment in an unstable state after test](#leave-environment-in-an-unstable-state-after-test) - [Test coverage analysis](#test-coverage-analysis) -- [Building with extra plugins](#building-with-extra-plugins) +- [Testing with plugins](#testing-with-plugins) - [Environment misc](#environment-misc) # Requirements @@ -552,11 +552,17 @@ Apart from using Gradle, it is also possible to gain insight in code coverage us Please read your IDE documentation for how to attach a debugger to a JVM process. -# Building with extra plugins +# Testing with plugins -Additional plugins may be built alongside OpenSearch, where their dependency on OpenSearch will be substituted with the local OpenSearch build. To add your plugin, create a directory called `opensearch-extra` as a sibling of OpenSearch. Checkout your plugin underneath `opensearch-extra` and the build will automatically pick it up. You can verify the plugin is included as part of the build by checking the projects of the build. +To test a plugin with a custom build of OpenSearch, build OpenSearch and use the `customDistributionUrl` setting supported by each plugin to override the OpenSearch distribution. - ./gradlew projects +For example, in your OpenSearch repository assemble a custom distribution. + + ./gradlew :distribution:archives:linux-tar:assemble + +Then in your plugin repository, substitute in your OpenSearch build + + ./gradlew run -PcustomDistributionUrl="/distribution/archives/linux-tar/build/distributions/opensearch-min-3.0.0-SNAPSHOT-linux-x64.tar.gz" # Environment misc From aaa92aeeb75a745ee7b058cc4b30b734e861cfc4 Mon Sep 17 00:00:00 2001 From: "Samuel.G" <1148690954@qq.com> Date: Sat, 14 Sep 2024 04:54:02 +0800 Subject: [PATCH 017/108] Fix case insensitive query on wildcard field (#15882) * fix case insensitive query on wildcard field Signed-off-by: gesong.samuel * fix YAML test Signed-off-by: gesong.samuel * add change log Signed-off-by: gesong.samuel --------- Signed-off-by: gesong.samuel Co-authored-by: gesong.samuel --- CHANGELOG.md | 2 +- .../search/270_wildcard_fieldtype_queries.yml | 41 +++++++++++++++++-- .../index/mapper/WildcardFieldMapper.java | 15 +++++-- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3268852cc99f9..ec4ac7504b9c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - +- Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) ### Security [Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.17...2.x diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml index c449a06cf633d..d92538824232d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml @@ -56,6 +56,12 @@ setup: id: 6 body: other_field: "test" + - do: + index: + index: test + id: 7 + body: + my_field: "ABCD" - do: indices.refresh: {} @@ -90,8 +96,9 @@ setup: query: term: my_field.lower: "abcd" - - match: { hits.total.value: 1 } + - match: { hits.total.value: 2 } - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "7" } - do: search: @@ -100,8 +107,9 @@ setup: query: term: my_field.lower: "ABCD" - - match: { hits.total.value: 1 } + - match: { hits.total.value: 2 } - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "7" } - do: search: @@ -215,7 +223,7 @@ setup: wildcard: my_field: value: "*" - - match: { hits.total.value: 5 } + - match: { hits.total.value: 6 } --- "regexp match-all works": - do: @@ -226,7 +234,7 @@ setup: regexp: my_field: value: ".*" - - match: { hits.total.value: 5 } + - match: { hits.total.value: 6 } --- "terms query on wildcard field matches": - do: @@ -237,3 +245,28 @@ setup: terms: { my_field: ["AbCd"] } - match: { hits.total.value: 1 } - match: { hits.hits.0._id: "5" } +--- +"case insensitive query on wildcard field": + - do: + search: + index: test + body: + query: + wildcard: + my_field: + value: "AbCd" + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "5" } + + - do: + search: + index: test + body: + query: + wildcard: + my_field: + value: "AbCd" + case_insensitive: true + - match: { hits.total.value: 2 } + - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "7" } diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index 0cb416a9b8370..e43e3bda692e7 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -40,6 +40,7 @@ import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.lucene.Lucene; +import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.analysis.IndexAnalyzers; @@ -464,7 +465,7 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo return existsQuery(context); } } else { - approximation = matchAllTermsQuery(name(), requiredNGrams); + approximation = matchAllTermsQuery(name(), requiredNGrams, caseInsensitive); } return new WildcardMatchingQuery(name(), approximation, matchPredicate, value, context, this); } @@ -678,7 +679,7 @@ public Query termsQuery(List values, QueryShardContext context) { StringBuilder pattern = new StringBuilder(); for (Object value : values) { String stringVal = BytesRefs.toString(value); - builder.add(matchAllTermsQuery(name(), getRequiredNGrams(stringVal)), BooleanClause.Occur.SHOULD); + builder.add(matchAllTermsQuery(name(), getRequiredNGrams(stringVal), false), BooleanClause.Occur.SHOULD); expectedValues.add(stringVal); if (pattern.length() > 0) { pattern.append('|'); @@ -688,10 +689,16 @@ public Query termsQuery(List values, QueryShardContext context) { return new WildcardMatchingQuery(name(), builder.build(), expectedValues::contains, pattern.toString(), context, this); } - private static BooleanQuery matchAllTermsQuery(String fieldName, Set terms) { + private static BooleanQuery matchAllTermsQuery(String fieldName, Set terms, boolean caseInsensitive) { BooleanQuery.Builder matchAllTermsBuilder = new BooleanQuery.Builder(); + Query query; for (String term : terms) { - matchAllTermsBuilder.add(new TermQuery(new Term(fieldName, term)), BooleanClause.Occur.FILTER); + if (caseInsensitive) { + query = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, term)); + } else { + query = new TermQuery(new Term(fieldName, term)); + } + matchAllTermsBuilder.add(query, BooleanClause.Occur.FILTER); } return matchAllTermsBuilder.build(); } From bd26056669f41826a500f944869b198149c476a2 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:58:07 +0530 Subject: [PATCH 018/108] Remove TooManyShardsInSnapshotsStatusException (#15877) Signed-off-by: Lakshya Taragi --- .../opensearch/OpenSearchServerException.java | 8 --- ...oManyShardsInSnapshotsStatusException.java | 69 ------------------- .../ExceptionSerializationTests.java | 2 - 3 files changed, 79 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index a1fc61834f69b..b0ab62259d5c0 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -1210,14 +1210,6 @@ public static void registerExceptions() { V_2_17_0 ) ); - registerExceptionHandle( - new OpenSearchExceptionHandle( - org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException.class, - org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException::new, - 175, - V_2_17_0 - ) - ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.cluster.block.IndexCreateBlockException.class, diff --git a/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java b/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java deleted file mode 100644 index 1689b3e4941ec..0000000000000 --- a/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.snapshots; - -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.rest.RestStatus; - -import java.io.IOException; - -/** - * Thrown if the number of shards across the requested resources (snapshot(s) or the index/indices of a particular snapshot) - * breaches the limit of snapshot.max_shards_allowed_in_status_api cluster setting - * - * @opensearch.internal - */ -public class TooManyShardsInSnapshotsStatusException extends SnapshotException { - - public TooManyShardsInSnapshotsStatusException( - final String repositoryName, - final SnapshotId snapshotId, - final String message, - final Throwable cause - ) { - super(repositoryName, snapshotId, message, cause); - } - - public TooManyShardsInSnapshotsStatusException(final String repositoryName, final String message, String... snapshotName) { - super(repositoryName, String.join(", ", snapshotName), message); - } - - public TooManyShardsInSnapshotsStatusException(StreamInput in) throws IOException { - super(in); - } - - @Override - public RestStatus status() { - return RestStatus.REQUEST_ENTITY_TOO_LARGE; - } -} diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index eff312a36dbc6..2e4a2d7bdd59c 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -119,7 +119,6 @@ import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInProgressException; import org.opensearch.snapshots.SnapshotInUseDeletionException; -import org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; import org.opensearch.transport.ActionNotFoundTransportException; @@ -899,7 +898,6 @@ public void testIds() { ids.put(172, ViewNotFoundException.class); ids.put(173, ViewAlreadyExistsException.class); ids.put(174, InvalidIndexContextException.class); - ids.put(175, TooManyShardsInSnapshotsStatusException.class); ids.put(10001, IndexCreateBlockException.class); Map, Integer> reverse = new HashMap<>(); From c709400543a16d7f7d6d644f26bb2f7a10b90caf Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Mon, 16 Sep 2024 15:28:29 +0530 Subject: [PATCH 019/108] Revert "Mute failing snapshot status tests (#15652)" (#15876) This reverts commit 48cf5f061094d89aaa564b97667f4965556a8e49. Signed-off-by: Lakshya Taragi --- .../test/snapshot.status/10_basic.yml | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml index 819f04407b219..c35f2419bdc91 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -25,40 +25,38 @@ setup: snapshot: test_snapshot wait_for_completion: true -# TODO: fix and unmute tests + - do: + snapshot.status: + repository: test_repo_status_1 + snapshot: test_snapshot + + - is_true: snapshots + - match: { snapshots.0.snapshot: test_snapshot } + - match: { snapshots.0.state: SUCCESS } + - gt: { snapshots.0.stats.incremental.file_count: 0 } + - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 } + - gt: { snapshots.0.stats.total.file_count: 0 } + - gt: { snapshots.0.stats.total.size_in_bytes: 0 } + - is_true: snapshots.0.stats.start_time_in_millis +## fast in memory snapshots can take less than one millisecond to complete. + - gte: { snapshots.0.stats.time_in_millis: 0 } + +--- +"Get missing snapshot status throws an exception": + + - do: + catch: /snapshot_missing_exception.+ is missing/ + snapshot.status: + repository: test_repo_status_1 + snapshot: test_nonexistent_snapshot + +--- +"Get missing snapshot status succeeds when ignoreUnavailable is true": + + - do: + snapshot.status: + repository: test_repo_status_1 + snapshot: test_nonexistent_snapshot + ignore_unavailable: true -# - do: -# snapshot.status: -# repository: test_repo_status_1 -# snapshot: test_snapshot -# -# - is_true: snapshots -# - match: { snapshots.0.snapshot: test_snapshot } -# - match: { snapshots.0.state: SUCCESS } -# - gt: { snapshots.0.stats.incremental.file_count: 0 } -# - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 } -# - gt: { snapshots.0.stats.total.file_count: 0 } -# - gt: { snapshots.0.stats.total.size_in_bytes: 0 } -# - is_true: snapshots.0.stats.start_time_in_millis -### fast in memory snapshots can take less than one millisecond to complete. -# - gte: { snapshots.0.stats.time_in_millis: 0 } -# -#--- -#"Get missing snapshot status throws an exception": -# -# - do: -# catch: /snapshot_missing_exception.+ is missing/ -# snapshot.status: -# repository: test_repo_status_1 -# snapshot: test_nonexistent_snapshot -# -#--- -#"Get missing snapshot status succeeds when ignoreUnavailable is true": -# -# - do: -# snapshot.status: -# repository: test_repo_status_1 -# snapshot: test_nonexistent_snapshot -# ignore_unavailable: true -# -# - is_true: snapshots + - is_true: snapshots From 45a8ed41f0c67f8853e36d95c1cc05d921b92739 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:09:17 -0400 Subject: [PATCH 020/108] Bump com.gradle.develocity from 3.18 to 3.18.1 (#15947) Bumps com.gradle.develocity from 3.18 to 3.18.1. --- updated-dependencies: - dependency-name: com.gradle.develocity dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index b79c2aee135fc..8412d198a2a29 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,7 @@ */ plugins { - id "com.gradle.develocity" version "3.18" + id "com.gradle.develocity" version "3.18.1" } ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE') From e3bbc74868dcba47b4231c8d6e095c2721d019e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:40:39 -0400 Subject: [PATCH 021/108] Bump ch.qos.logback:logback-core from 1.5.6 to 1.5.8 in /test/fixtures/hdfs-fixture (#15946) * Bump ch.qos.logback:logback-core in /test/fixtures/hdfs-fixture Bumps [ch.qos.logback:logback-core](https://github.com/qos-ch/logback) from 1.5.6 to 1.5.8. - [Commits](https://github.com/qos-ch/logback/compare/v_1.5.6...v_1.5.8) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec4ac7504b9c2..cc313fdcdb849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) +- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) ### Changed diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index b5cd12ef0c11f..20fd1058a181d 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -74,7 +74,7 @@ dependencies { api 'org.apache.zookeeper:zookeeper:3.9.2' api "org.apache.commons:commons-text:1.12.0" api "commons-net:commons-net:3.11.1" - api "ch.qos.logback:logback-core:1.5.6" + api "ch.qos.logback:logback-core:1.5.8" api "ch.qos.logback:logback-classic:1.2.13" api "org.jboss.xnio:xnio-nio:3.8.16.Final" api 'org.jline:jline:3.26.3' From 81288b1044348d99cce6d23770755a464121f7cc Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 18 Sep 2024 00:28:59 +0530 Subject: [PATCH 022/108] remote publication checksum stats (#15957) * Remote publication checksum stats Signed-off-by: Himshikha Gupta Co-authored-by: Himshikha Gupta --- .../remote/RemoteStatePublicationIT.java | 19 ++++++++++ .../remote/RemoteClusterStateService.java | 6 ++++ .../gateway/remote/RemoteDownloadStats.java | 36 +++++++++++++++++++ .../remote/RemotePersistenceStats.java | 24 ++++++++++--- .../RemoteClusterStateServiceTests.java | 10 ++++++ 5 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index faab3645ae894..ffb9352e8ba47 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -56,6 +56,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteDownloadStats.CHECKSUM_VALIDATION_FAILED_COUNT; import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA; import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA; @@ -405,10 +406,28 @@ private void assertDataNodeDownloadStats(NodesStatsResponse nodesStatsResponse) assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getSuccessCount() > 0); assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getFailedCount()); assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getTotalTimeInMillis() > 0); + assertEquals( + 0, + dataNodeDiscoveryStats.getClusterStateStats() + .getPersistenceStats() + .get(0) + .getExtendedFields() + .get(CHECKSUM_VALIDATION_FAILED_COUNT) + .get() + ); assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getSuccessCount() > 0); assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getFailedCount()); assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getTotalTimeInMillis() > 0); + assertEquals( + 0, + dataNodeDiscoveryStats.getClusterStateStats() + .getPersistenceStats() + .get(1) + .getExtendedFields() + .get(CHECKSUM_VALIDATION_FAILED_COUNT) + .get() + ); } private Map getMetadataFiles(BlobStoreRepository repository, String subDirectory) throws IOException { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 12d10fd908b44..ece29180f9cf5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -1644,6 +1644,12 @@ void validateClusterStateFromChecksum( failedValidation ) ); + if (isFullStateDownload) { + remoteStateStats.stateFullDownloadValidationFailed(); + } else { + remoteStateStats.stateDiffDownloadValidationFailed(); + } + if (isFullStateDownload && remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.FAILURE)) { throw new IllegalStateException( "Cluster state checksums do not match during full state read. Validation failed for " + failedValidation diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java new file mode 100644 index 0000000000000..a8f4b33a19c37 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.cluster.coordination.PersistedStateStats; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Download stats for remote state + * + * @opensearch.internal + */ +public class RemoteDownloadStats extends PersistedStateStats { + static final String CHECKSUM_VALIDATION_FAILED_COUNT = "checksum_validation_failed_count"; + private AtomicLong checksumValidationFailedCount = new AtomicLong(0); + + public RemoteDownloadStats(String statsName) { + super(statsName); + addToExtendedFields(CHECKSUM_VALIDATION_FAILED_COUNT, checksumValidationFailedCount); + } + + public void checksumValidationFailedCount() { + checksumValidationFailedCount.incrementAndGet(); + } + + public long getChecksumValidationFailedCount() { + return checksumValidationFailedCount.get(); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java index 417ebdafd3ba7..11f26ac8b3ed9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java @@ -18,16 +18,16 @@ public class RemotePersistenceStats { RemoteUploadStats remoteUploadStats; - PersistedStateStats remoteDiffDownloadStats; - PersistedStateStats remoteFullDownloadStats; + RemoteDownloadStats remoteDiffDownloadStats; + RemoteDownloadStats remoteFullDownloadStats; final String FULL_DOWNLOAD_STATS = "remote_full_download"; final String DIFF_DOWNLOAD_STATS = "remote_diff_download"; public RemotePersistenceStats() { remoteUploadStats = new RemoteUploadStats(); - remoteDiffDownloadStats = new PersistedStateStats(DIFF_DOWNLOAD_STATS); - remoteFullDownloadStats = new PersistedStateStats(FULL_DOWNLOAD_STATS); + remoteDiffDownloadStats = new RemoteDownloadStats(DIFF_DOWNLOAD_STATS); + remoteFullDownloadStats = new RemoteDownloadStats(FULL_DOWNLOAD_STATS); } public void cleanUpAttemptFailed() { @@ -90,6 +90,22 @@ public void stateDiffDownloadFailed() { remoteDiffDownloadStats.stateFailed(); } + public void stateDiffDownloadValidationFailed() { + remoteDiffDownloadStats.checksumValidationFailedCount(); + } + + public void stateFullDownloadValidationFailed() { + remoteFullDownloadStats.checksumValidationFailedCount(); + } + + public long getStateDiffDownloadValidationFailed() { + return remoteDiffDownloadStats.getChecksumValidationFailedCount(); + } + + public long getStateFullDownloadValidationFailed() { + return remoteFullDownloadStats.getChecksumValidationFailedCount(); + } + public PersistedStateStats getUploadStats() { return remoteUploadStats; } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index b11d5e48bec06..56857285fa8d3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -3342,6 +3342,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullC anyString(), anyBoolean() ); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws IOException { @@ -3374,6 +3375,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws ); mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); verify(mockService, times(1)).validateClusterStateFromChecksum(manifest, clusterState, ClusterName.DEFAULT.value(), NODE_ID, true); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationModeNone() throws IOException { @@ -3406,6 +3408,7 @@ public void testGetClusterStateForManifestWithChecksumValidationModeNone() throw ); mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); verify(mockService, times(0)).validateClusterStateFromChecksum(any(), any(), anyString(), anyString(), anyBoolean()); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMismatch() throws IOException { @@ -3448,6 +3451,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma NODE_ID, true ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatch() throws IOException { @@ -3494,6 +3498,7 @@ public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatc NODE_ID, true ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksum() throws IOException { @@ -3535,6 +3540,7 @@ public void testGetClusterStateUsingDiffWithChecksum() throws IOException { eq(NODE_ID), eq(false) ); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOException { @@ -3576,6 +3582,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOExceptio eq(NODE_ID), eq(false) ); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws IOException { @@ -3616,6 +3623,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws I eq(NODE_ID), eq(false) ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws IOException { @@ -3677,6 +3685,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws I eq(NODE_ID), eq(false) ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOException { @@ -3738,6 +3747,7 @@ public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOExceptio eq(NODE_ID), eq(false) ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { From 8347d0ec22aa865dea3bc4b2027fb7bdf486fab2 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Wed, 18 Sep 2024 03:12:05 +0800 Subject: [PATCH 023/108] Add validation for the search backpressure cancellation settings (#15501) * Fix updating search backpressure settings crashing the cluster Signed-off-by: Gao Binlong * Modify change log Signed-off-by: Gao Binlong * Fix version check Signed-off-by: Gao Binlong * Increase test coverage Signed-off-by: Gao Binlong * Format the code Signed-off-by: Gao Binlong * Optimize some code Signed-off-by: Gao Binlong * Fix typo Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 2 + .../test/cluster.put_settings/10_basic.yml | 113 ++++++++++++++++++ .../opensearch/common/settings/Setting.java | 13 ++ .../SearchBackpressureService.java | 6 +- .../backpressure/SearchBackpressureState.java | 5 +- .../settings/SearchBackpressureSettings.java | 16 ++- .../settings/SearchShardTaskSettings.java | 16 ++- .../settings/SearchTaskSettings.java | 16 ++- .../common/settings/SettingTests.java | 28 +++++ .../SearchBackpressureSettingsTests.java | 28 +++++ 10 files changed, 231 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc313fdcdb849..db41e2eee16f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) +- Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) + ### Security [Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.17...2.x diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index 107d298b597d3..0f4a45e4591a3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -98,3 +98,116 @@ - match: {error.root_cause.0.type: "illegal_argument_exception"} - match: { error.root_cause.0.reason: "Invalid SearchBackpressureMode: monitor-only" } - match: { status: 400 } + + +--- +"Test setting search backpressure cancellation settings": + - skip: + version: "- 2.99.99" + reason: "Fixed in 3.0.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_burst: 11 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_burst: "11"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_rate: 0.1 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_rate: "0.1"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_ratio: 0.2 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_ratio: "0.2"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_burst: 12 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_burst: "12"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_rate: 0.3 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_rate: "0.3"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_ratio: 0.4 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_ratio: "0.4"} + +--- +"Test setting invalid search backpressure cancellation_rate and cancellation_ratio": + - skip: + version: "- 2.99.99" + reason: "Fixed in 3.0.0" + + - do: + catch: /search_backpressure.search_task.cancellation_rate must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_rate: 0.0 + + - do: + catch: /search_backpressure.search_task.cancellation_ratio must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_ratio: 0.0 + + - do: + catch: /search_backpressure.search_shard_task.cancellation_rate must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_rate: 0.0 + + - do: + catch: /search_backpressure.search_shard_task.cancellation_ratio must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_ratio: 0.0 diff --git a/server/src/main/java/org/opensearch/common/settings/Setting.java b/server/src/main/java/org/opensearch/common/settings/Setting.java index fea4c165809ba..081029c1c106c 100644 --- a/server/src/main/java/org/opensearch/common/settings/Setting.java +++ b/server/src/main/java/org/opensearch/common/settings/Setting.java @@ -1855,6 +1855,10 @@ public static Setting doubleSetting( ); } + public static Setting doubleSetting(String key, double defaultValue, Validator validator, Property... properties) { + return new Setting<>(key, Double.toString(defaultValue), Double::parseDouble, validator, properties); + } + /** * A writeable parser for double * @@ -1961,6 +1965,15 @@ public static Setting doubleSetting( ); } + public static Setting doubleSetting( + String key, + Setting fallbackSetting, + Validator validator, + Property... properties + ) { + return new Setting<>(new SimpleKey(key), fallbackSetting, fallbackSetting::getRaw, Double::parseDouble, validator, properties); + } + /// simpleString public static Setting simpleString(String key, Property... properties) { diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java index 43b9f8ae87529..e98046ba1dede 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java @@ -158,14 +158,16 @@ public SearchBackpressureService( timeNanosSupplier, getSettings().getSearchTaskSettings().getCancellationRateNanos(), getSettings().getSearchTaskSettings().getCancellationBurst(), - getSettings().getSearchTaskSettings().getCancellationRatio() + getSettings().getSearchTaskSettings().getCancellationRatio(), + getSettings().getSearchTaskSettings().getCancellationRate() ), SearchShardTask.class, new SearchBackpressureState( timeNanosSupplier, getSettings().getSearchShardTaskSettings().getCancellationRateNanos(), getSettings().getSearchShardTaskSettings().getCancellationBurst(), - getSettings().getSearchShardTaskSettings().getCancellationRatio() + getSettings().getSearchShardTaskSettings().getCancellationRatio(), + getSettings().getSearchShardTaskSettings().getCancellationRate() ) ); this.settings.getSearchTaskSettings().addListener(searchBackpressureStates.get(SearchTask.class)); diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java index 5f086bd498036..36f5b25e002c3 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java @@ -43,12 +43,15 @@ public class SearchBackpressureState implements CancellationSettingsListener { LongSupplier timeNanosSupplier, double cancellationRateNanos, double cancellationBurst, - double cancellationRatio + double cancellationRatio, + double cancellationRate ) { rateLimiter = new AtomicReference<>(new TokenBucket(timeNanosSupplier, cancellationRateNanos, cancellationBurst)); ratioLimiter = new AtomicReference<>(new TokenBucket(this::getCompletionCount, cancellationRatio, cancellationBurst)); this.timeNanosSupplier = timeNanosSupplier; this.cancellationBurst = cancellationBurst; + this.cancellationRatio = cancellationRatio; + this.cancellationRate = cancellationRate; } public long getCompletionCount() { diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java index 79494eb0d3c24..55a031382f282 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java @@ -61,8 +61,14 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATIO = Setting.doubleSetting( "search_backpressure.cancellation_ratio", Defaults.CANCELLATION_RATIO, - 0.0, - 1.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.cancellation_ratio must be > 0"); + } + if (value > 1.0) { + throw new IllegalArgumentException("search_backpressure.cancellation_ratio must be <= 1.0"); + } + }, Setting.Property.Deprecated, Setting.Property.Dynamic, Setting.Property.NodeScope @@ -78,7 +84,11 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATE = Setting.doubleSetting( "search_backpressure.cancellation_rate", Defaults.CANCELLATION_RATE, - 0.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.cancellation_rate must be > 0"); + } + }, Setting.Property.Deprecated, Setting.Property.Dynamic, Setting.Property.NodeScope diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java index 6d016c7466362..38213506c55b7 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java @@ -44,8 +44,14 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATIO = Setting.doubleSetting( "search_backpressure.search_shard_task.cancellation_ratio", SearchBackpressureSettings.SETTING_CANCELLATION_RATIO, - 0.0, - 1.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_ratio must be > 0"); + } + if (value > 1.0) { + throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_ratio must be <= 1.0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -58,7 +64,11 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATE = Setting.doubleSetting( "search_backpressure.search_shard_task.cancellation_rate", SearchBackpressureSettings.SETTING_CANCELLATION_RATE, - 0.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_rate must be > 0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java index 4b34323b1ddc6..f9af7f9b59fdb 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java @@ -48,8 +48,14 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATIO = Setting.doubleSetting( "search_backpressure.search_task.cancellation_ratio", Defaults.CANCELLATION_RATIO, - 0.0, - 1.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_task.cancellation_ratio must be > 0"); + } + if (value > 1.0) { + throw new IllegalArgumentException("search_backpressure.search_task.cancellation_ratio must be <= 1.0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -62,7 +68,11 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATE = Setting.doubleSetting( "search_backpressure.search_task.cancellation_rate", Defaults.CANCELLATION_RATE, - 0.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_task.cancellation_rate must be > 0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); diff --git a/server/src/test/java/org/opensearch/common/settings/SettingTests.java b/server/src/test/java/org/opensearch/common/settings/SettingTests.java index 7ebee680e8e52..c3c399a9d88b2 100644 --- a/server/src/test/java/org/opensearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SettingTests.java @@ -1274,6 +1274,20 @@ public void testFloatParser() throws Exception { public void testDoubleWithDefaultValue() { Setting doubleSetting = Setting.doubleSetting("foo.bar", 42.1); assertEquals(doubleSetting.get(Settings.EMPTY), Double.valueOf(42.1)); + + Setting doubleSettingWithValidator = Setting.doubleSetting("foo.bar", 42.1, value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("The setting foo.bar must be >0"); + } + }); + try { + assertThrows( + IllegalArgumentException.class, + () -> doubleSettingWithValidator.get(Settings.builder().put("foo.bar", randomFrom(-1, 0)).build()) + ); + } catch (IllegalArgumentException ex) { + assertEquals("The setting foo.bar must be >0", ex.getMessage()); + } } public void testDoubleWithFallbackValue() { @@ -1282,6 +1296,20 @@ public void testDoubleWithFallbackValue() { assertEquals(doubleSetting.get(Settings.EMPTY), Double.valueOf(2.1)); assertEquals(doubleSetting.get(Settings.builder().put("foo.bar", 3.2).build()), Double.valueOf(3.2)); assertEquals(doubleSetting.get(Settings.builder().put("foo.baz", 3.2).build()), Double.valueOf(3.2)); + + Setting doubleSettingWithValidator = Setting.doubleSetting("foo.bar", fallbackSetting, value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("The setting foo.bar must be >0"); + } + }); + try { + assertThrows( + IllegalArgumentException.class, + () -> doubleSettingWithValidator.get(Settings.builder().put("foo.bar", randomFrom(-1, 0)).build()) + ); + } catch (IllegalArgumentException ex) { + assertEquals("The setting foo.bar must be >0", ex.getMessage()); + } } public void testDoubleWithMinMax() throws Exception { diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java index a02ca3cf877ad..683ada76c7683 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java @@ -37,4 +37,32 @@ public void testSearchBackpressureSettingValidateInvalidMode() { () -> new SearchBackpressureSettings(settings, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) ); } + + public void testInvalidCancellationRate() { + Settings settings1 = Settings.builder().put("search_backpressure.search_task.cancellation_rate", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings1, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + + Settings settings2 = Settings.builder().put("search_backpressure.search_shard_task.cancellation_rate", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings2, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + } + + public void testInvalidCancellationRatio() { + Settings settings1 = Settings.builder().put("search_backpressure.search_task.cancellation_ratio", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings1, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + + Settings settings2 = Settings.builder().put("search_backpressure.search_shard_task.cancellation_ratio", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings2, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + } } From eb5b70370e548cd0f3b51721397df818b3050daa Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 17 Sep 2024 15:44:32 -0400 Subject: [PATCH 024/108] Fix flaky terminaton conditions for org.opensearch.rest.ReactorNetty4StreamingStressIT.testCloseClientStreamingRequest test case (#15959) Signed-off-by: Andriy Redko --- .../org/opensearch/rest/ReactorNetty4StreamingStressIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java index a853ed56fad32..9da456f618ffc 100644 --- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java +++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java @@ -8,6 +8,7 @@ package org.opensearch.rest; +import org.apache.hc.core5.http.ConnectionClosedException; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.StreamingRequest; @@ -75,7 +76,7 @@ public void testCloseClientStreamingRequest() throws Exception { } }) .then(() -> scheduler.advanceTimeBy(delay)) - .expectErrorMatches(t -> t instanceof InterruptedIOException) + .expectErrorMatches(t -> t instanceof InterruptedIOException || t instanceof ConnectionClosedException) .verify(); } } From 7c427d9fd588597343654d2f534952f7116d1df4 Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:29:12 -0700 Subject: [PATCH 025/108] Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder (#15916) Signed-off-by: David Zane --- CHANGELOG.md | 1 + .../support/ValuesSourceAggregationBuilder.java | 10 +++++++++- .../org/opensearch/search/sort/FieldSortBuilder.java | 8 +++++++- .../bucket/range/RangeAggregationBuilderTests.java | 1 + .../opensearch/search/sort/FieldSortBuilderTests.java | 1 + 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db41e2eee16f5..309d9cb06cc94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) +- Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java index 7a73fafb4a809..1ccceb1d77dcb 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java @@ -40,6 +40,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.WithFieldName; import org.opensearch.script.Script; import org.opensearch.search.aggregations.AbstractAggregationBuilder; import org.opensearch.search.aggregations.AggregationInitializationException; @@ -57,7 +58,9 @@ * * @opensearch.internal */ -public abstract class ValuesSourceAggregationBuilder> extends AbstractAggregationBuilder { +public abstract class ValuesSourceAggregationBuilder> extends AbstractAggregationBuilder + implements + WithFieldName { public static void declareFields( AbstractObjectParser, T> objectParser, @@ -292,6 +295,11 @@ public String field() { return field; } + @Override + public String fieldName() { + return field(); + } + /** * Sets the script to use for this aggregation. */ diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java index 5cecda1346b90..9825b2cbbe08e 100644 --- a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java @@ -65,6 +65,7 @@ import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; +import org.opensearch.index.query.WithFieldName; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.SearchSortValuesAndFormats; @@ -86,7 +87,7 @@ * * @opensearch.internal */ -public class FieldSortBuilder extends SortBuilder { +public class FieldSortBuilder extends SortBuilder implements WithFieldName { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(FieldSortBuilder.class); public static final String NAME = "field_sort"; @@ -184,6 +185,11 @@ public String getFieldName() { return this.fieldName; } + @Override + public String fieldName() { + return getFieldName(); + } + /** * Sets the value when a field is missing in a doc. Can also be set to {@code _last} or * {@code _first} to sort missing last or first respectively. diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java index 4362ce48003cc..14532e30f8984 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java @@ -128,6 +128,7 @@ public void testNumericKeys() throws IOException { ); assertThat(builder.getName(), equalTo("test")); assertThat(builder.field(), equalTo("f")); + assertThat(builder.fieldName(), equalTo("f")); assertThat(builder.ranges, equalTo(List.of(new RangeAggregator.Range("1", null, 0d)))); } } diff --git a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java index 9b8cd1b5f1ce0..ced952db555aa 100644 --- a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java @@ -196,6 +196,7 @@ protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse()); if (expectedType == SortField.Type.CUSTOM) { assertEquals(builder.getFieldName(), sortField.getField()); + assertEquals(builder.fieldName(), sortField.getField()); } assertEquals(DocValueFormat.RAW, format); } From b2a7136722e9972edcf14ff03abd4f86ae4923b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:58:08 +0800 Subject: [PATCH 026/108] Bump com.microsoft.azure:msal4j from 1.17.0 to 1.17.1 in /plugins/repository-azure (#15945) * Bump com.microsoft.azure:msal4j in /plugins/repository-azure Bumps [com.microsoft.azure:msal4j](https://github.com/AzureAD/microsoft-authentication-library-for-java) from 1.17.0 to 1.17.1. - [Release notes](https://github.com/AzureAD/microsoft-authentication-library-for-java/releases) - [Changelog](https://github.com/AzureAD/microsoft-authentication-library-for-java/blob/dev/changelog.txt) - [Commits](https://github.com/AzureAD/microsoft-authentication-library-for-java/commits) --- updated-dependencies: - dependency-name: com.microsoft.azure:msal4j dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 | 1 - plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 309d9cb06cc94..4ed85b630bbd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) +- Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) ### Changed diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 5669c5e559ac8..616da0be62c30 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -61,7 +61,7 @@ dependencies { // Start of transitive dependencies for azure-identity api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0' api "net.java.dev.jna:jna-platform:${versions.jna}" - api 'com.microsoft.azure:msal4j:1.17.0' + api 'com.microsoft.azure:msal4j:1.17.1' api 'com.nimbusds:oauth2-oidc-sdk:11.19.1' api 'com.nimbusds:nimbus-jose-jwt:9.40' api 'com.nimbusds:content-type:2.3' diff --git a/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 deleted file mode 100644 index 34101c989eecd..0000000000000 --- a/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7d37157da92b719f250b0023234ac9dda922a2a5 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 new file mode 100644 index 0000000000000..46c14e819b630 --- /dev/null +++ b/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 @@ -0,0 +1 @@ +4eb31a9919d9b103c548af7e37e6f9d9f6e46dbc \ No newline at end of file From 4802d0d744cfd19827ab61c0807f557b768bd1aa Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 18 Sep 2024 07:57:38 -0400 Subject: [PATCH 027/108] Update version in README to 2.18.0 (#15968) Signed-off-by: Craig Perkins --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d4a9a671c013..9294d739a0103 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability") [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) -[![2.17.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.0") +[![2.18.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0") [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) From 3937ccb123aa5ced3c963ec2eef5e07c2085b004 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 15:47:49 -0700 Subject: [PATCH 028/108] [AUTO] [main] Add bwc version 2.17.1. (#15966) * Resolve readme conflicts Signed-off-by: Kunal Kotwani * Fix version identifier Signed-off-by: Kunal Kotwani --------- Signed-off-by: Kunal Kotwani Co-authored-by: Kunal Kotwani --- .ci/bwcVersions | 1 + README.md | 1 + libs/core/src/main/java/org/opensearch/Version.java | 1 + 3 files changed, 3 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 0f59fec3d824a..ccb47c1a3b724 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -38,4 +38,5 @@ BWC_VERSION: - "2.16.0" - "2.16.1" - "2.17.0" + - "2.17.1" - "2.18.0" diff --git a/README.md b/README.md index 9294d739a0103..03728aee0135c 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) [![2.18.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0") +[![2.17.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.1") [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index ca5dd306cf907..b86b8459fb8a8 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -109,6 +109,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_16_1 = new Version(2160199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1); + public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version CURRENT = V_3_0_0; From 3a1b6d1795f23f2d14c4f2174867ce70ed88525e Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:14:00 -0700 Subject: [PATCH 029/108] Refactor branch deletion for backports (#15983) Signed-off-by: Sayali Gaikawad --- .github/workflows/delete_backport_branch.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index 387a124b8cb6a..b60c2225db26a 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -7,9 +7,16 @@ on: jobs: delete-branch: runs-on: ubuntu-latest + permissions: + contents: write if: startsWith(github.event.pull_request.head.ref,'backport/') steps: - name: Delete merged branch - uses: SvanBoxel/delete-merged-branch@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: actions/github-script@v5 + with: + script: | + github.rest.git.deleteRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `heads/${context.payload.pull_request.head.ref}`, + }) From 80ff07eb0c5485832e6fcbc46a9d100435ed3051 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Thu, 19 Sep 2024 12:50:22 +0530 Subject: [PATCH 030/108] star tree file formats refactoring and fixing offset bug (#15975) Signed-off-by: Bharathwaj G --- .../startree/builder/BaseStarTreeBuilder.java | 31 +++--- .../builder/OffHeapStarTreeBuilder.java | 4 +- .../builder/OnHeapStarTreeBuilder.java | 4 +- .../builder/StarTreeDocsFileManager.java | 43 +++----- .../startree/index/StarTreeValues.java | 57 +++++----- .../utils/SequentialDocValuesIterator.java | 84 ++++++--------- .../SortedNumericStarTreeValuesIterator.java | 32 ++++++ .../iterator/StarTreeValuesIterator.java | 48 +++++++++ .../startree/utils/iterator/package-info.java | 14 +++ .../datacube/startree/StarTreeTestUtils.java | 8 +- .../builder/AbstractStarTreeBuilderTests.java | 101 +++++++++++------- .../SequentialDocValuesIteratorTests.java | 64 +++++------ 12 files changed, 281 insertions(+), 209 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index a1d638616f2aa..5e5814528fcd2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -34,6 +34,7 @@ import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.FieldMapper; import org.opensearch.index.mapper.FieldValueConverter; @@ -193,7 +194,9 @@ public List getMetricReaders(SegmentWriteState stat metricFieldInfo = getFieldInfo(metric.getField(), DocValuesType.SORTED_NUMERIC); } metricReader = new SequentialDocValuesIterator( - fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + new SortedNumericStarTreeValuesIterator( + fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + ) ); } metricReaders.add(metricReader); @@ -228,7 +231,7 @@ public void build( dimensionFieldInfo = getFieldInfo(dimension, DocValuesType.SORTED_NUMERIC); } dimensionReaders[i] = new SequentialDocValuesIterator( - fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo) + new SortedNumericStarTreeValuesIterator(fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo)) ); } Iterator starTreeDocumentIterator = sortAndAggregateSegmentDocuments(dimensionReaders, metricReaders); @@ -287,7 +290,7 @@ void appendDocumentsToStarTree(Iterator starTreeDocumentIterat } } - private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDocs) throws IOException { + private void serializeStarTree(int numSegmentStarTreeDocuments, int numStarTreeDocs) throws IOException { // serialize the star tree data long dataFilePointer = dataOut.getFilePointer(); StarTreeWriter starTreeWriter = new StarTreeWriter(); @@ -299,7 +302,7 @@ private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDo starTreeField, metricAggregatorInfos, numStarTreeNodes, - numSegmentStarTreeDocument, + numSegmentStarTreeDocuments, numStarTreeDocs, dataFilePointer, totalStarTreeDataLength @@ -400,22 +403,20 @@ protected StarTreeDocument getStarTreeDocument( ) throws IOException { Long[] dims = new Long[numDimensions]; int i = 0; - for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { - dimensionDocValueIterator.nextDoc(currentDocId); - Long val = dimensionDocValueIterator.value(currentDocId); + for (SequentialDocValuesIterator dimensionValueIterator : dimensionReaders) { + dimensionValueIterator.nextEntry(currentDocId); + Long val = dimensionValueIterator.value(currentDocId); dims[i] = val; i++; } i = 0; Object[] metrics = new Object[metricReaders.size()]; - for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { - metricDocValuesIterator.nextDoc(currentDocId); + for (SequentialDocValuesIterator metricValuesIterator : metricReaders) { + metricValuesIterator.nextEntry(currentDocId); // As part of merge, we traverse the star tree doc values // The type of data stored in metric fields is different from the // actual indexing field they're based on - metrics[i] = metricAggregatorInfos.get(i) - .getValueAggregators() - .toAggregatedValueType(metricDocValuesIterator.value(currentDocId)); + metrics[i] = metricAggregatorInfos.get(i).getValueAggregators().toAggregatedValueType(metricValuesIterator.value(currentDocId)); i++; } return new StarTreeDocument(dims, metrics); @@ -502,7 +503,7 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIte for (int i = 0; i < numDimensions; i++) { if (dimensionReaders[i] != null) { try { - dimensionReaders[i].nextDoc(currentDocId); + dimensionReaders[i].nextEntry(currentDocId); } catch (IOException e) { logger.error("unable to iterate to next doc", e); throw new RuntimeException("unable to iterate to next doc", e); @@ -530,7 +531,7 @@ private Object[] getStarTreeMetricsFromSegment(int currentDocId, List mergeStarTrees(List starTreeValuesSub .size()]; for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } List metricReaders = new ArrayList<>(); // get doc id set iterators for metrics @@ -164,7 +164,7 @@ Iterator mergeStarTrees(List starTreeValuesSub metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } int currentDocId = 0; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 1a5c906ad413b..13c6d03c4dc3d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -138,7 +138,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } List metricReaders = new ArrayList<>(); @@ -150,7 +150,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java index 779ed77b0540a..15ed153249243 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java @@ -21,7 +21,6 @@ import java.io.Closeable; import java.io.IOException; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -55,11 +54,9 @@ public class StarTreeDocsFileManager extends AbstractDocumentsFileManager implem private RandomAccessInput starTreeDocsFileRandomInput; private IndexOutput starTreeDocsFileOutput; private final Map fileToEndDocIdMap; - private final List starTreeDocumentOffsets = new ArrayList<>(); private int currentFileStartDocId; private int numReadableStarTreeDocuments; private int starTreeFileCount = -1; - private int currBytes = 0; private final int fileCountMergeThreshold; private int numStarTreeDocs = 0; @@ -98,7 +95,11 @@ IndexOutput createStarTreeDocumentsFileOutput() throws IOException { public void writeStarTreeDocument(StarTreeDocument starTreeDocument, boolean isAggregatedDoc) throws IOException { assert isAggregatedDoc == true; int numBytes = writeStarTreeDocument(starTreeDocument, starTreeDocsFileOutput, true); - addStarTreeDocumentOffset(numBytes); + if (docSizeInBytes == -1) { + docSizeInBytes = numBytes; + } else { + assert docSizeInBytes == numBytes; + } numStarTreeDocs++; } @@ -106,7 +107,14 @@ public void writeStarTreeDocument(StarTreeDocument starTreeDocument, boolean isA public StarTreeDocument readStarTreeDocument(int docId, boolean isAggregatedDoc) throws IOException { assert isAggregatedDoc == true; ensureDocumentReadable(docId); - return readStarTreeDocument(starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId), true); + return readStarTreeDocument(starTreeDocsFileRandomInput, getOffset(docId), true); + } + + /** + * Returns offset for the docId based on the current file start id + */ + private long getOffset(int docId) { + return (long) (docId - currentFileStartDocId) * docSizeInBytes; } @Override @@ -119,19 +127,10 @@ public Long getDimensionValue(int docId, int dimensionId) throws IOException { public Long[] readDimensions(int docId) throws IOException { ensureDocumentReadable(docId); Long[] dims = new Long[starTreeField.getDimensionsOrder().size()]; - readDimensions(dims, starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId)); + readDimensions(dims, starTreeDocsFileRandomInput, getOffset(docId)); return dims; } - private void addStarTreeDocumentOffset(int bytes) { - starTreeDocumentOffsets.add(currBytes); - currBytes += bytes; - if (docSizeInBytes == -1) { - docSizeInBytes = bytes; - } - assert docSizeInBytes == bytes; - } - /** * Load the correct StarTreeDocuments file based on the docId */ @@ -199,7 +198,6 @@ private void loadStarTreeDocumentFile(int docId) throws IOException { * If the operation is only for reading existing documents, a new file is not created. */ private void closeAndMaybeCreateNewFile(boolean shouldCreateFileForAppend, int numStarTreeDocs) throws IOException { - currBytes = 0; if (starTreeDocsFileOutput != null) { fileToEndDocIdMap.put(starTreeDocsFileOutput.getName(), numStarTreeDocs); IOUtils.close(starTreeDocsFileOutput); @@ -232,7 +230,6 @@ private void mergeFiles(int numStarTreeDocs) throws IOException { deleteOldFiles(); fileToEndDocIdMap.clear(); fileToEndDocIdMap.put(mergedOutput.getName(), numStarTreeDocs); - resetStarTreeDocumentOffsets(); } } @@ -259,17 +256,6 @@ private void deleteOldFiles() throws IOException { } } - /** - * Reset the star tree document offsets based on the merged file - */ - private void resetStarTreeDocumentOffsets() { - int curr = 0; - for (int i = 0; i < starTreeDocumentOffsets.size(); i++) { - starTreeDocumentOffsets.set(i, curr); - curr += docSizeInBytes; - } - } - @Override public void close() { try { @@ -288,7 +274,6 @@ public void close() { tmpDirectory.deleteFile(file); } catch (IOException ignored) {} // similar to IOUtils.deleteFilesIgnoringExceptions } - starTreeDocumentOffsets.clear(); fileToEndDocIdMap.clear(); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index a34bbbe9ee738..255ad343cde32 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexInput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.CompositeIndexMetadata; @@ -25,6 +24,8 @@ import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import java.io.IOException; import java.util.ArrayList; @@ -59,14 +60,14 @@ public class StarTreeValues implements CompositeIndexValues { private final StarTreeNode root; /** - * A map containing suppliers for DocIdSetIterators for dimensions. + * A map containing suppliers for StarTreeValues iterators for dimensions. */ - private final Map> dimensionDocValuesIteratorMap; + private final Map> dimensionValuesIteratorMap; /** - * A map containing suppliers for DocIdSetIterators for metrics. + * A map containing suppliers for StarTreeValues iterators for metrics. */ - private final Map> metricDocValuesIteratorMap; + private final Map> metricValuesIteratorMap; /** * A map containing attributes associated with the star tree values. @@ -84,22 +85,22 @@ public class StarTreeValues implements CompositeIndexValues { * * @param starTreeField The StarTreeField object representing the star tree field configuration. * @param root The root node of the star tree. - * @param dimensionDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for dimensions. - * @param metricDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for metrics. + * @param dimensionValuesIteratorMap A map containing suppliers for StarTreeValues iterators for dimensions. + * @param metricValuesIteratorMap A map containing suppliers for StarTreeValues iterators for metrics. * @param attributes A map containing attributes associated with the star tree values. */ public StarTreeValues( StarTreeField starTreeField, StarTreeNode root, - Map> dimensionDocValuesIteratorMap, - Map> metricDocValuesIteratorMap, + Map> dimensionValuesIteratorMap, + Map> metricValuesIteratorMap, Map attributes, StarTreeMetadata compositeIndexMetadata ) { this.starTreeField = starTreeField; this.root = root; - this.dimensionDocValuesIteratorMap = dimensionDocValuesIteratorMap; - this.metricDocValuesIteratorMap = metricDocValuesIteratorMap; + this.dimensionValuesIteratorMap = dimensionValuesIteratorMap; + this.metricValuesIteratorMap = metricValuesIteratorMap; this.attributes = attributes; this.starTreeMetadata = compositeIndexMetadata; } @@ -146,12 +147,12 @@ public StarTreeValues( this.root = StarTreeFactory.createStarTree(compositeIndexDataIn, starTreeMetadata); // get doc id set iterators for metrics and dimensions - dimensionDocValuesIteratorMap = new LinkedHashMap<>(); - metricDocValuesIteratorMap = new LinkedHashMap<>(); + dimensionValuesIteratorMap = new LinkedHashMap<>(); + metricValuesIteratorMap = new LinkedHashMap<>(); // get doc id set iterators for dimensions for (String dimension : starTreeMetadata.getDimensionFields()) { - dimensionDocValuesIteratorMap.put(dimension, () -> { + dimensionValuesIteratorMap.put(dimension, () -> { try { SortedNumericDocValues dimensionSortedNumericDocValues = null; if (readState != null) { @@ -162,9 +163,9 @@ public StarTreeValues( dimensionSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(dimensionfieldInfo); } } - return getSortedNumericDocValues(dimensionSortedNumericDocValues); + return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(dimensionSortedNumericDocValues)); } catch (IOException e) { - throw new RuntimeException("Error loading dimension DocIdSetIterator", e); + throw new RuntimeException("Error loading dimension StarTreeValuesIterator", e); } }); } @@ -177,7 +178,7 @@ public StarTreeValues( metric.getField(), metricStat.getTypeName() ); - metricDocValuesIteratorMap.put(metricFullName, () -> { + metricValuesIteratorMap.put(metricFullName, () -> { try { SortedNumericDocValues metricSortedNumericDocValues = null; if (readState != null) { @@ -186,7 +187,7 @@ public StarTreeValues( metricSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(metricFieldInfo); } } - return getSortedNumericDocValues(metricSortedNumericDocValues); + return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(metricSortedNumericDocValues)); } catch (IOException e) { throw new RuntimeException("Error loading metric DocIdSetIterator", e); } @@ -239,30 +240,30 @@ public Map getAttributes() { } /** - * Returns the DocIdSetIterator for the specified dimension. + * Returns the StarTreeValues iterator for the specified dimension. * * @param dimension The name of the dimension. - * @return The DocIdSetIterator for the specified dimension. + * @return The StarTreeValuesIterator for the specified dimension. */ - public DocIdSetIterator getDimensionDocIdSetIterator(String dimension) { + public StarTreeValuesIterator getDimensionValuesIterator(String dimension) { - if (dimensionDocValuesIteratorMap.containsKey(dimension)) { - return dimensionDocValuesIteratorMap.get(dimension).get(); + if (dimensionValuesIteratorMap.containsKey(dimension)) { + return dimensionValuesIteratorMap.get(dimension).get(); } throw new IllegalArgumentException("dimension [" + dimension + "] does not exist in the segment."); } /** - * Returns the DocIdSetIterator for the specified fully qualified metric name. + * Returns the StarTreeValues iterator for the specified fully qualified metric name. * * @param fullyQualifiedMetricName The fully qualified name of the metric. - * @return The DocIdSetIterator for the specified fully qualified metric name. + * @return The StarTreeValuesIterator for the specified fully qualified metric name. */ - public DocIdSetIterator getMetricDocIdSetIterator(String fullyQualifiedMetricName) { + public StarTreeValuesIterator getMetricValuesIterator(String fullyQualifiedMetricName) { - if (metricDocValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { - return metricDocValuesIteratorMap.get(fullyQualifiedMetricName).get(); + if (metricValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { + return metricValuesIteratorMap.get(fullyQualifiedMetricName).get(); } throw new IllegalArgumentException("metric [" + fullyQualifiedMetricName + "] does not exist in the segment."); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java index 061841d3e140a..9029a451ca4d9 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java @@ -9,15 +9,19 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import java.io.IOException; /** - * Coordinates the reading of documents across multiple DocIdSetIterators. - * It encapsulates a single DocIdSetIterator and maintains the latest document ID and its associated value. + * Coordinates the reading of documents across multiple StarTreeValuesIterator. + * It encapsulates a single StarTreeValuesIterator and maintains the latest document ID and its associated value. + * + * In case of merge , this will be reading the entries of star tree values and in case of flush this will go through + * the actual segment documents. + * * @opensearch.experimental */ @ExperimentalApi @@ -26,76 +30,56 @@ public class SequentialDocValuesIterator { /** * The doc id set iterator associated for each field. */ - private final DocIdSetIterator docIdSetIterator; + private final StarTreeValuesIterator starTreeValuesIterator; /** - * The value associated with the latest document. + * The id of the latest record/entry. */ - private Long docValue; + private int entryId = -1; - /** - * The id of the latest document. - */ - private int docId = -1; - - /** - * Constructs a new SequentialDocValuesIterator instance with the given DocIdSetIterator. - * - * @param docIdSetIterator the DocIdSetIterator to be associated with this instance - */ - public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) { - this.docIdSetIterator = docIdSetIterator; - } - - /** - * Returns the id of the latest document. - * - * @return the id of the latest document - */ - public int getDocId() { - return docId; + public SequentialDocValuesIterator(StarTreeValuesIterator starTreeValuesIterator) { + this.starTreeValuesIterator = starTreeValuesIterator; } /** - * Sets the id of the latest document. + * Returns the ID of the star tree record/entry or the segment document id * - * @param docId the ID of the latest document + * @return the ID of the star tree record/entry or the segment document id */ - private void setDocId(int docId) { - this.docId = docId; + public int getEntryId() { + return entryId; } /** - * Returns the DocIdSetIterator associated with this instance. + * Sets the id of the latest entry. * - * @return the DocIdSetIterator associated with this instance + * @param entryId the ID of the star tree record/entry or the segment document id */ - public DocIdSetIterator getDocIdSetIterator() { - return docIdSetIterator; + private void setEntryId(int entryId) { + this.entryId = entryId; } - public int nextDoc(int currentDocId) throws IOException { + public int nextEntry(int currentEntryId) throws IOException { // if doc id stored is less than or equal to the requested doc id , return the stored doc id - if (docId >= currentDocId) { - return docId; + if (entryId >= currentEntryId) { + return entryId; } - setDocId(this.docIdSetIterator.nextDoc()); - return docId; + setEntryId(this.starTreeValuesIterator.nextEntry()); + return entryId; } - public Long value(int currentDocId) throws IOException { - if (this.getDocIdSetIterator() instanceof SortedNumericDocValues) { - SortedNumericDocValues sortedNumericDocValues = (SortedNumericDocValues) this.getDocIdSetIterator(); - if (currentDocId < 0) { - throw new IllegalStateException("invalid doc id to fetch the next value"); + public Long value(int currentEntryId) throws IOException { + if (starTreeValuesIterator instanceof SortedNumericStarTreeValuesIterator) { + if (currentEntryId < 0) { + throw new IllegalStateException("invalid entry id to fetch the next value"); } - if (currentDocId == DocIdSetIterator.NO_MORE_DOCS) { - throw new IllegalStateException("DocValuesIterator is already exhausted"); + if (currentEntryId == StarTreeValuesIterator.NO_MORE_ENTRIES) { + throw new IllegalStateException("StarTreeValuesIterator is already exhausted"); } - if (docId == DocIdSetIterator.NO_MORE_DOCS || docId != currentDocId) { + if (entryId == StarTreeValuesIterator.NO_MORE_ENTRIES || entryId != currentEntryId) { return null; } - return sortedNumericDocValues.nextValue(); + return ((SortedNumericStarTreeValuesIterator) starTreeValuesIterator).nextValue(); } else { throw new IllegalStateException("Unsupported Iterator requested for SequentialDocValuesIterator"); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java new file mode 100644 index 0000000000000..27afdf1479b4e --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; + +/** + * Wrapper iterator class for StarTree index to traverse through SortedNumericDocValues + * + * @opensearch.experimental + */ +@ExperimentalApi +public class SortedNumericStarTreeValuesIterator extends StarTreeValuesIterator { + + public SortedNumericStarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { + super(docIdSetIterator); + } + + public long nextValue() throws IOException { + return ((SortedNumericDocValues) docIdSetIterator).nextValue(); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java new file mode 100644 index 0000000000000..32866f3e50092 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; + +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; + +/** + * Wrapper iterator class for StarTree index in place of DocIdSetIterator to read / traverse the docValues formats. + * This is needed since star tree values are different from segment documents and number of star tree values + * can even exceed segment docs in the worst cases. + * + * @opensearch.experimental + */ +@ExperimentalApi +public abstract class StarTreeValuesIterator { + + public static final int NO_MORE_ENTRIES = Integer.MAX_VALUE; + protected final DocIdSetIterator docIdSetIterator; + + public StarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { + this.docIdSetIterator = docIdSetIterator; + } + + public int entryId() { + return docIdSetIterator.docID(); + } + + public int nextEntry() throws IOException { + return docIdSetIterator.nextDoc(); + } + + public int advance(int target) throws IOException { + return docIdSetIterator.advance(target); + } + + public long cost() { + return docIdSetIterator.cost(); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java new file mode 100644 index 0000000000000..3c6444a4a5cac --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This contains classes for StarTreeValues iterators + * + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index b7395b993f67b..7cae1cd25ee93 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -54,7 +54,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } List metricReaders = new ArrayList<>(); @@ -69,7 +69,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } @@ -92,7 +92,7 @@ public static StarTreeDocument getStarTreeDocument( Long[] dims = new Long[dimensionReaders.length]; int i = 0; for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { - dimensionDocValueIterator.nextDoc(currentDocId); + dimensionDocValueIterator.nextEntry(currentDocId); Long val = dimensionDocValueIterator.value(currentDocId); dims[i] = val; i++; @@ -100,7 +100,7 @@ public static StarTreeDocument getStarTreeDocument( i = 0; Object[] metrics = new Object[metricReaders.size()]; for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { - metricDocValuesIterator.nextDoc(currentDocId); + metricDocValuesIterator.nextEntry(currentDocId); metrics[i] = toAggregatorValueType(metricDocValuesIterator.value(currentDocId), fieldValueConverters.get(i)); i++; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 65adc43ea8bea..b77200f173e71 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -53,6 +53,8 @@ import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.FieldValueConverter; @@ -395,7 +397,9 @@ SequentialDocValuesIterator[] getDimensionIterators(StarTreeDocument[] starTreeD docsWithField.add(i); } } - sequentialDocValuesIterators[j] = new SequentialDocValuesIterator(getSortedNumericMock(dimList, docsWithField)); + sequentialDocValuesIterators[j] = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)) + ); } return sequentialDocValuesIterators; } @@ -412,7 +416,9 @@ List getMetricIterators(StarTreeDocument[] starTree docsWithField.add(i); } } - sequentialDocValuesIterators.add(new SequentialDocValuesIterator(getSortedNumericMock(metricslist, docsWithField))); + sequentialDocValuesIterators.add( + new SequentialDocValuesIterator(new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricslist, docsWithField))) + ); } return sequentialDocValuesIterators; } @@ -1985,10 +1991,16 @@ public void testFlushFlow() throws IOException { List metricsWithField = List.of(0, 1, 2, 3, 4, 5); compositeField = getStarTreeFieldWithMultipleMetrics(); - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); writeState = getWriteState(6, writeState.segmentInfo.getId()); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); @@ -2081,10 +2093,16 @@ public void testFlushFlowDimsReverse() throws IOException { List metricsWithField = List.of(0, 1, 2, 3, 4, 5); compositeField = getStarTreeFieldWithMultipleMetrics(); - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); writeState = getWriteState(6, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( @@ -2508,9 +2526,14 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d1sndv = dimList; SortedNumericDocValues d2sndv = dimList2; SortedNumericDocValues m1sndv = metricsList; - Map> dimDocIdSetIterators = Map.of("field1", () -> d1sndv, "field3", () -> d2sndv); + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(d2sndv) + ); - Map> metricDocIdSetIterators = new LinkedHashMap<>(); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); for (Metric metric : sf.getMetrics()) { for (MetricStat metricStat : metric.getMetrics()) { String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -2518,7 +2541,7 @@ private StarTreeValues getStarTreeValues( metric.getField(), metricStat.getTypeName() ); - metricDocIdSetIterators.put(metricFullName, () -> m1sndv); + metricDocIdSetIterators.put(metricFullName, () -> new SortedNumericStarTreeValuesIterator(m1sndv)); } } @@ -3648,18 +3671,18 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList1, metricsWithField1); - Map> dimDocIdSetIterators = Map.of( + Map> dimDocIdSetIterators = Map.of( "field1", - () -> d1sndv, + () -> new SortedNumericStarTreeValuesIterator(d1sndv), "field3", - () -> d2sndv, + () -> new SortedNumericStarTreeValuesIterator(d2sndv), "field5", - () -> d3sndv, + () -> new SortedNumericStarTreeValuesIterator(d3sndv), "field8", - () -> d4sndv + () -> new SortedNumericStarTreeValuesIterator(d4sndv) ); - Map> metricDocIdSetIterators = new LinkedHashMap<>(); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); metricDocIdSetIterators.put( fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -3667,7 +3690,7 @@ private StarTreeValues getStarTreeValues( "field2", sf.getMetrics().get(0).getMetrics().get(0).getTypeName() ), - () -> m1sndv + () -> new SortedNumericStarTreeValuesIterator(m1sndv) ); metricDocIdSetIterators.put( fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -3675,7 +3698,7 @@ private StarTreeValues getStarTreeValues( "_doc_count", sf.getMetrics().get(1).getMetrics().get(0).getTypeName() ), - () -> m2sndv + () -> new SortedNumericStarTreeValuesIterator(m2sndv) ); // metricDocIdSetIterators.put("field2", () -> m1sndv); // metricDocIdSetIterators.put("_doc_count", () -> m2sndv); @@ -4093,24 +4116,24 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric(); - Map> dimDocIdSetIterators = Map.of( + Map> dimDocIdSetIterators = Map.of( "field1", - () -> d1sndv, + () -> new SortedNumericStarTreeValuesIterator(d1sndv), "field3", - () -> d2sndv, + () -> new SortedNumericStarTreeValuesIterator(d2sndv), "field5", - () -> d3sndv, + () -> new SortedNumericStarTreeValuesIterator(d3sndv), "field8", - () -> d4sndv + () -> new SortedNumericStarTreeValuesIterator(d4sndv) ); - Map> metricDocIdSetIterators = Map.of( + Map> metricDocIdSetIterators = Map.of( "sf_field2_sum_metric", - () -> m1sndv, + () -> new SortedNumericStarTreeValuesIterator(m1sndv), "sf_field2_value_count_metric", - () -> valucountsndv, + () -> new SortedNumericStarTreeValuesIterator(valucountsndv), "sf__doc_count_doc_count_metric", - () -> m2sndv + () -> new SortedNumericStarTreeValuesIterator(m2sndv) ); StarTreeValues starTreeValues = new StarTreeValues( @@ -4129,24 +4152,24 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues f2valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric(); - Map> f2dimDocIdSetIterators = Map.of( + Map> f2dimDocIdSetIterators = Map.of( "field1", - () -> f2d1sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d1sndv), "field3", - () -> f2d2sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d2sndv), "field5", - () -> f2d3sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d3sndv), "field8", - () -> f2d4sndv + () -> new SortedNumericStarTreeValuesIterator(f2d4sndv) ); - Map> f2metricDocIdSetIterators = Map.of( + Map> f2metricDocIdSetIterators = Map.of( "sf_field2_sum_metric", - () -> f2m1sndv, + () -> new SortedNumericStarTreeValuesIterator(f2m1sndv), "sf_field2_value_count_metric", - () -> f2valucountsndv, + () -> new SortedNumericStarTreeValuesIterator(f2valucountsndv), "sf__doc_count_doc_count_metric", - () -> f2m2sndv + () -> new SortedNumericStarTreeValuesIterator(f2m2sndv) ); StarTreeValues starTreeValues2 = new StarTreeValues( compositeField, diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java index f56f7d9906ae1..78d63800abd16 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java @@ -9,15 +9,13 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.BytesRef; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.test.OpenSearchTestCase; import org.junit.BeforeClass; @@ -59,50 +57,32 @@ public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); - SequentialDocValuesIterator result = new SequentialDocValuesIterator(producer.getSortedNumeric(mockFieldInfo)); - assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass()); - } - - public void testCreateIterator_UnsupportedType() throws IOException { - DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); - BinaryDocValues iterator = Mockito.mock(BinaryDocValues.class); - when(producer.getBinary(mockFieldInfo)).thenReturn(iterator); - SequentialDocValuesIterator result = new SequentialDocValuesIterator(producer.getBinary(mockFieldInfo)); - assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass()); - when(iterator.nextDoc()).thenReturn(0); - when(iterator.binaryValue()).thenReturn(new BytesRef("123")); + SequentialDocValuesIterator result = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(producer.getSortedNumeric(mockFieldInfo)) + ); - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { - result.nextDoc(0); - result.value(0); - }); - assertEquals("Unsupported Iterator requested for SequentialDocValuesIterator", exception.getMessage()); } public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(iterator.nextDoc()).thenReturn(0); when(iterator.nextValue()).thenReturn(123L); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); - sequentialDocValuesIterator.nextDoc(0); + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator) + ); + sequentialDocValuesIterator.nextEntry(0); long result = sequentialDocValuesIterator.value(0); assertEquals(123L, result); } - public void testGetNextValue_UnsupportedIterator() { - DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); - - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { sequentialDocValuesIterator.value(0); }); - assertEquals("Unsupported Iterator requested for SequentialDocValuesIterator", exception.getMessage()); - } - - public void testNextDoc() throws IOException { + public void testNextEntry() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator) + ); when(iterator.nextDoc()).thenReturn(5); - int result = sequentialDocValuesIterator.nextDoc(5); + int result = sequentialDocValuesIterator.nextEntry(5); assertEquals(5, result); } @@ -110,8 +90,12 @@ public void test_multipleCoordinatedDocumentReader() throws IOException { SortedNumericDocValues iterator1 = Mockito.mock(SortedNumericDocValues.class); SortedNumericDocValues iterator2 = Mockito.mock(SortedNumericDocValues.class); - SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator(iterator1); - SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator(iterator2); + SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator1) + ); + SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator2) + ); when(iterator1.nextDoc()).thenReturn(0); when(iterator2.nextDoc()).thenReturn(1); @@ -119,13 +103,13 @@ public void test_multipleCoordinatedDocumentReader() throws IOException { when(iterator1.nextValue()).thenReturn(9L); when(iterator2.nextValue()).thenReturn(9L); - sequentialDocValuesIterator1.nextDoc(0); - sequentialDocValuesIterator2.nextDoc(0); - assertEquals(0, sequentialDocValuesIterator1.getDocId()); + sequentialDocValuesIterator1.nextEntry(0); + sequentialDocValuesIterator2.nextEntry(0); + assertEquals(0, sequentialDocValuesIterator1.getEntryId()); assertEquals(9L, (long) sequentialDocValuesIterator1.value(0)); assertNull(sequentialDocValuesIterator2.value(0)); - assertNotEquals(0, sequentialDocValuesIterator2.getDocId()); - assertEquals(1, sequentialDocValuesIterator2.getDocId()); + assertNotEquals(0, sequentialDocValuesIterator2.getEntryId()); + assertEquals(1, sequentialDocValuesIterator2.getEntryId()); assertEquals(9L, (long) sequentialDocValuesIterator2.value(1)); } } From 6bef1e1d8e2dda8d88c2c8f2a7875b492239a480 Mon Sep 17 00:00:00 2001 From: kkewwei Date: Thu, 19 Sep 2024 16:48:52 +0800 Subject: [PATCH 031/108] Fix infinite loop in nested agg (#15931) Signed-off-by: kkewwei --- CHANGELOG.md | 1 + .../search.aggregation/410_nested_aggs.yml | 111 ++++++++++++++++++ .../opensearch/index/mapper/ObjectMapper.java | 13 ++ .../bucket/nested/NestedAggregator.java | 14 +-- .../index/mapper/ObjectMapperTests.java | 47 ++++++++ 5 files changed, 174 insertions(+), 12 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ed85b630bbd4..636684af939bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) - Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) +- Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml new file mode 100644 index 0000000000000..c115dd4751f8f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml @@ -0,0 +1,111 @@ +--- +# The test setup includes: +# - Create nested mapping for test_nested_agg_index index +# - Index two example documents +# - nested agg + +setup: + - do: + indices.create: + index: test_nested_agg_index + body: + mappings: + properties: + a: + type: nested + properties: + b1: + type: keyword + b2: + type: nested + properties: + c: + type: nested + properties: + d: + type: keyword + + - do: + bulk: + refresh: true + body: | + {"index": {"_index": "test_nested_agg_index", "_id": "0"}} + {"a": { "b1": "b11", "b2": { "c": { "d": "d1" } }}} + {"index": {"_index": "test_nested_agg_index", "_id": "1"}} + {"a": { "b1": "b12", "b2": { "c": { "d": "d2" } }}} + +--- +# Delete Index when connection is teardown +teardown: + - do: + indices.delete: + index: test_nested_agg_index + +--- +"Supported queries": + - skip: + version: " - 2.17.99" + reason: "fixed in 2.18.0" + + # Verify Document Count + - do: + search: + body: { + query: { + match_all: { } + } + } + + - length: { hits.hits: 2 } + + # Verify nested aggregation + - do: + search: + body: { + aggs: { + nested_agg: { + nested: { + path: "a" + }, + aggs: { + a_b1: { + terms: { + field: "a.b1" + }, + aggs: { + "c": { + nested: { + path: "a.b2.c" + }, + aggs: { + "d": { + terms: { + field: "a.b2.c.d" + } + } + } + } + } + } + } + } + } + } + + - length: { hits.hits: 2 } + - match: { aggregations.nested_agg.doc_count: 2 } + - length: { aggregations.nested_agg.a_b1.buckets: 2 } + + - match: { aggregations.nested_agg.a_b1.buckets.0.key: "b11" } + - match: { aggregations.nested_agg.a_b1.buckets.0.doc_count: 1 } + - match: { aggregations.nested_agg.a_b1.buckets.0.c.doc_count: 1 } + - length: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets: "1" } + - match: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets.0.key: "d1" } + - match: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets.0.doc_count: 1 } + + - match: { aggregations.nested_agg.a_b1.buckets.1.key: "b12" } + - match: { aggregations.nested_agg.a_b1.buckets.1.doc_count: 1 } + - match: { aggregations.nested_agg.a_b1.buckets.1.c.doc_count: 1 } + - length: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets: "1" } + - match: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets.0.key: "d2" } + - match: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets.0.doc_count: 1 } diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index dd984373fc9df..b93c82d7a5c7c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -171,6 +171,19 @@ public void setIncludeInParent(boolean value) { public void setIncludeInRoot(boolean value) { includeInRoot = new Explicit<>(value, true); } + + public static boolean isParent(ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, MapperService mapperService) { + if (parentObjectMapper == null || childObjectMapper == null) { + return false; + } + + ObjectMapper parent = childObjectMapper.getParentObjectMapper(mapperService); + while (parent != null && parent != parentObjectMapper) { + childObjectMapper = parent; + parent = childObjectMapper.getParentObjectMapper(mapperService); + } + return parentObjectMapper == parent; + } } /** diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java index 150efa878f866..db8979d611b4f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java @@ -46,7 +46,6 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.ParseField; -import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ObjectMapper; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; @@ -63,6 +62,8 @@ import java.util.List; import java.util.Map; +import static org.opensearch.index.mapper.ObjectMapper.Nested.isParent; + /** * Aggregate all docs that match a nested path * @@ -98,17 +99,6 @@ public class NestedAggregator extends BucketsAggregator implements SingleBucketA this.collectsFromSingleBucket = cardinality.map(estimate -> estimate < 2); } - private boolean isParent(ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, MapperService mapperService) { - if (parentObjectMapper == null) { - return false; - } - ObjectMapper parent; - do { - parent = childObjectMapper.getParentObjectMapper(mapperService); - } while (parent != null && parent != parentObjectMapper); - return parentObjectMapper == parent; - } - @Override public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException { IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(ctx); diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index cb06bf23d9cbe..b415e1e657f7f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -51,8 +51,12 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; + +import org.mockito.Mockito; import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; +import static org.opensearch.index.mapper.ObjectMapper.Nested.isParent; import static org.hamcrest.Matchers.containsString; public class ObjectMapperTests extends OpenSearchSingleNodeTestCase { @@ -568,6 +572,49 @@ public void testCompositeFields() throws Exception { FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } + public void testNestedIsParent() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("a") + .field("type", "nested") + .startObject("properties") + .field("b1", Collections.singletonMap("type", "keyword")) + .startObject("b2") + .field("type", "nested") + .startObject("properties") + .startObject("c") + .field("type", "nested") + .startObject("properties") + .field("d", Collections.singletonMap("type", "keyword")) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + + DocumentMapper documentMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + MapperService mapperService = Mockito.mock(MapperService.class); + Mockito.when(mapperService.getObjectMapper(("a"))).thenReturn(documentMapper.objectMappers().get("a")); + Mockito.when(mapperService.getObjectMapper(("a.b2"))).thenReturn(documentMapper.objectMappers().get("a.b2")); + Mockito.when(mapperService.getObjectMapper(("a.b2.c"))).thenReturn(documentMapper.objectMappers().get("a.b2.c")); + + assertTrue(isParent(documentMapper.objectMappers().get("a"), documentMapper.objectMappers().get("a.b2.c"), mapperService)); + assertTrue(isParent(documentMapper.objectMappers().get("a"), documentMapper.objectMappers().get("a.b2"), mapperService)); + assertTrue(isParent(documentMapper.objectMappers().get("a.b2"), documentMapper.objectMappers().get("a.b2.c"), mapperService)); + + assertFalse(isParent(documentMapper.objectMappers().get("a.b2.c"), documentMapper.objectMappers().get("a"), mapperService)); + assertFalse(isParent(documentMapper.objectMappers().get("a.b2"), documentMapper.objectMappers().get("a"), mapperService)); + assertFalse(isParent(documentMapper.objectMappers().get("a.b2.c"), documentMapper.objectMappers().get("a.b2"), mapperService)); + } + @Override protected Collection> getPlugins() { return pluginList(InternalSettingsPlugin.class); From 620db0a64950cbd6a38642f0be1f592060dac1ee Mon Sep 17 00:00:00 2001 From: SwethaGuptha <156877431+SwethaGuptha@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:03:02 +0530 Subject: [PATCH 032/108] Explicitly set include discovery nodes flag in reloadSecureSetting request. (#15972) Signed-off-by: Swetha Guptha --- .../cluster/node/reload/NodesReloadSecureSettingsRequest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java index aab7ea54f87c2..d694721feda38 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java @@ -60,7 +60,7 @@ public class NodesReloadSecureSettingsRequest extends BaseNodesRequest Date: Thu, 19 Sep 2024 20:48:36 +0800 Subject: [PATCH 033/108] Update version check in yaml test file for bitmap filtering (#15993) Signed-off-by: Gao Binlong --- .../rest-api-spec/test/search/370_bitmap_filtering.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml index d728070adb188..c885e3fbc2446 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml @@ -1,7 +1,7 @@ --- setup: - skip: - version: " - 2.99.99" + version: " - 2.16.99" reason: The bitmap filtering feature is available in 2.17 and later. - do: indices.create: From 77ddfd640cfde672d103c7ff9f25521362cea14e Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> Date: Thu, 19 Sep 2024 13:50:47 -0700 Subject: [PATCH 034/108] Fix deleting backport branches workflow (#16003) Signed-off-by: Sayali Gaikawad --- .github/workflows/delete_backport_branch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index b60c2225db26a..91626a0bc5092 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -11,7 +11,7 @@ jobs: contents: write if: startsWith(github.event.pull_request.head.ref,'backport/') steps: - - name: Delete merged branch + - name: Delete merged branch uses: actions/github-script@v5 with: script: | From 1bc81d3eee07e96b6c6393510b41d194276e204c Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 19 Sep 2024 16:55:21 -0400 Subject: [PATCH 035/108] Remove identity-related feature flagged code from the RestController (#15430) * Add authenticate to IdentityPlugin interface Signed-off-by: Craig Perkins * Handle null Signed-off-by: Craig Perkins * Fix tests Signed-off-by: Craig Perkins * Fix ActionModuleTests Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Add DelegatingRestHandlerTests Signed-off-by: Craig Perkins * Address forbiddenApi Signed-off-by: Craig Perkins * Remove authenticate from IdentityPlugin and keep RestController feature flagged code removed Signed-off-by: Craig Perkins * Move RestTokenExtractor to identity-shiro plugin Signed-off-by: Craig Perkins * Remove change in IdentityService Signed-off-by: Craig Perkins * Remove changes in ActionModuleTests Signed-off-by: Craig Perkins * Add tests for RestTokenExtractor Signed-off-by: Craig Perkins * Remove DelegatingRestHandler Signed-off-by: Craig Perkins * Call super instead of keeping a reference to the delegated restHandler Signed-off-by: Craig Perkins * Address code review comments Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + .../identity/shiro/ShiroIdentityPlugin.java | 47 ++++++++++++++-- .../identity/shiro/ShiroSecurityManager.java | 2 - .../identity/shiro/ShiroSubject.java | 2 - .../identity/shiro/ShiroTokenExtractor.java | 8 +-- .../identity/shiro/ShiroTokenManager.java | 2 - .../shiro/realm/BCryptPasswordMatcher.java | 2 - .../identity/shiro/realm/OpenSearchRealm.java | 6 +-- .../opensearch/identity/shiro/realm/User.java | 2 - .../shiro/ShiroIdentityPluginTests.java | 14 ++--- .../shiro/ShiroTokenExtractorTests.java | 45 ++++++++++++++++ .../org/opensearch/action/ActionModule.java | 2 +- .../org/opensearch/rest/RestController.java | 54 +------------------ .../extensions/ExtensionsManagerTests.java | 3 +- .../opensearch/rest/RestControllerTests.java | 30 ++++------- .../rest/RestHttpResponseHeadersTests.java | 14 +---- .../indices/RestValidateQueryActionTests.java | 12 +---- .../test/rest/RestActionTestCase.java | 15 +----- 18 files changed, 118 insertions(+), 143 deletions(-) rename server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java => plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java (86%) create mode 100644 plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 636684af939bf..b5aef4bb551d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) +- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java index af802596ebaa7..2da788242a745 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java @@ -13,20 +13,28 @@ import org.apache.shiro.SecurityUtils; import org.apache.shiro.mgt.SecurityManager; import org.opensearch.client.Client; +import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.identity.PluginSubject; import org.opensearch.identity.Subject; +import org.opensearch.identity.tokens.AuthToken; import org.opensearch.identity.tokens.TokenManager; +import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.IdentityPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestHandler; +import org.opensearch.rest.RestRequest; import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; @@ -34,14 +42,12 @@ import java.util.Collection; import java.util.Collections; import java.util.function.Supplier; +import java.util.function.UnaryOperator; /** * Identity implementation with Shiro - * - * @opensearch.experimental */ -@ExperimentalApi -public final class ShiroIdentityPlugin extends Plugin implements IdentityPlugin { +public final class ShiroIdentityPlugin extends Plugin implements IdentityPlugin, ActionPlugin { private Logger log = LogManager.getLogger(this.getClass()); private final Settings settings; @@ -101,6 +107,37 @@ public TokenManager getTokenManager() { } @Override + public UnaryOperator getRestHandlerWrapper(ThreadContext threadContext) { + return AuthcRestHandler::new; + } + + class AuthcRestHandler extends RestHandler.Wrapper { + + public AuthcRestHandler(RestHandler original) { + super(original); + } + + @Override + public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { + try { + final AuthToken token = ShiroTokenExtractor.extractToken(request); + // If no token was found, continue executing the request + if (token == null) { + // Authentication did not fail so return true. Authorization is handled at the action level. + super.handleRequest(request, channel, client); + return; + } + ShiroSubject shiroSubject = (ShiroSubject) getCurrentSubject(); + shiroSubject.authenticate(token); + // Caller was authorized, forward the request to the handler + super.handleRequest(request, channel, client); + } catch (final Exception e) { + final BytesRestResponse bytesRestResponse = new BytesRestResponse(RestStatus.UNAUTHORIZED, e.getMessage()); + channel.sendResponse(bytesRestResponse); + } + } + } + public PluginSubject getPluginSubject(Plugin plugin) { return new ShiroPluginSubject(threadPool); } diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java index 96cf05ac53a1a..0a809dd6c9071 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java @@ -15,8 +15,6 @@ /** * OpenSearch specific security manager implementation - * - * @opensearch.experimental */ public class ShiroSecurityManager extends DefaultSecurityManager { diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java index 72a168f23c5cd..73ce3f835fc9b 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java @@ -17,8 +17,6 @@ /** * Subject backed by Shiro - * - * @opensearch.experimental */ public class ShiroSubject implements UserSubject { private final ShiroTokenManager authTokenHandler; diff --git a/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java similarity index 86% rename from server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java rename to plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java index 4bd3ebdded588..86be5ca664daa 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java @@ -5,11 +5,13 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.identity.tokens; +package org.opensearch.identity.shiro; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.core.common.Strings; +import org.opensearch.identity.tokens.AuthToken; +import org.opensearch.identity.tokens.BasicAuthToken; import org.opensearch.rest.RestRequest; import java.util.Collections; @@ -18,9 +20,9 @@ /** * Extracts tokens from RestRequests used for authentication */ -public class RestTokenExtractor { +public class ShiroTokenExtractor { - private static final Logger logger = LogManager.getLogger(RestTokenExtractor.class); + private static final Logger logger = LogManager.getLogger(ShiroTokenExtractor.class); public final static String AUTH_HEADER_NAME = "Authorization"; diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java index a14215aa7655b..cd54bbf9b3124 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java @@ -36,8 +36,6 @@ /** * Extracts Shiro's {@link AuthenticationToken} from different types of auth headers - * - * @opensearch.experimental */ class ShiroTokenManager implements TokenManager { diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java index a2cb78425929e..f8113101deb70 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java @@ -16,8 +16,6 @@ /** * Password matcher for BCrypt - * - * @opensearch.experimental */ public class BCryptPasswordMatcher implements CredentialsMatcher { diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java index ef405a5637ae7..1fc9a1f437a42 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java @@ -25,8 +25,6 @@ /** * Internal Realm is a custom realm using the internal OpenSearch IdP - * - * @opensearch.experimental */ public class OpenSearchRealm extends AuthenticatingRealm { private static final String DEFAULT_REALM_NAME = "internal"; @@ -93,7 +91,7 @@ public OpenSearchRealm build() { public User getInternalUser(final String principalIdentifier) throws UnknownAccountException { final User userRecord = internalUsers.get(principalIdentifier); if (userRecord == null) { - throw new UnknownAccountException(); + throw new UnknownAccountException("Incorrect credentials"); } return userRecord; } @@ -131,7 +129,7 @@ protected AuthenticationInfo doGetAuthenticationInfo(final AuthenticationToken t return sai; } else { // Bad password - throw new IncorrectCredentialsException(); + throw new IncorrectCredentialsException("Incorrect credentials"); } } diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java index 35b3348a955d7..1d2d0fed800e2 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java @@ -12,8 +12,6 @@ /** * A non-volatile and immutable object in the storage. - * - * @opensearch.experimental */ public class User { diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java index bc14410d81de0..a15538e48bd66 100644 --- a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java +++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java @@ -13,7 +13,7 @@ import org.opensearch.identity.IdentityService; import org.opensearch.plugins.IdentityPlugin; import org.opensearch.test.OpenSearchTestCase; -import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; import java.util.List; @@ -21,27 +21,27 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; public class ShiroIdentityPluginTests extends OpenSearchTestCase { public void testSingleIdentityPluginSucceeds() { - TestThreadPool threadPool = new TestThreadPool(getTestName()); IdentityPlugin identityPlugin1 = new ShiroIdentityPlugin(Settings.EMPTY); List pluginList1 = List.of(identityPlugin1); - IdentityService identityService1 = new IdentityService(Settings.EMPTY, threadPool, pluginList1); + IdentityService identityService1 = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), pluginList1); assertThat(identityService1.getTokenManager(), is(instanceOf(ShiroTokenManager.class))); - terminate(threadPool); } public void testMultipleIdentityPluginsFail() { - TestThreadPool threadPool = new TestThreadPool(getTestName()); IdentityPlugin identityPlugin1 = new ShiroIdentityPlugin(Settings.EMPTY); IdentityPlugin identityPlugin2 = new ShiroIdentityPlugin(Settings.EMPTY); IdentityPlugin identityPlugin3 = new ShiroIdentityPlugin(Settings.EMPTY); List pluginList = List.of(identityPlugin1, identityPlugin2, identityPlugin3); - Exception ex = assertThrows(OpenSearchException.class, () -> new IdentityService(Settings.EMPTY, threadPool, pluginList)); + Exception ex = assertThrows( + OpenSearchException.class, + () -> new IdentityService(Settings.EMPTY, mock(ThreadPool.class), pluginList) + ); assert (ex.getMessage().contains("Multiple identity plugins are not supported,")); - terminate(threadPool); } } diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java new file mode 100644 index 0000000000000..4dc398bacb707 --- /dev/null +++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.identity.shiro; + +import org.opensearch.identity.tokens.AuthToken; +import org.opensearch.identity.tokens.BasicAuthToken; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.rest.FakeRestRequest; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class ShiroTokenExtractorTests extends OpenSearchTestCase { + + public void testAuthorizationHeaderExtractionWithBasicAuthToken() { + String basicAuthHeader = Base64.getEncoder().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8)); + RestRequest fakeRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( + Map.of(ShiroTokenExtractor.AUTH_HEADER_NAME, List.of(BasicAuthToken.TOKEN_IDENTIFIER + " " + basicAuthHeader)) + ).build(); + AuthToken extractedToken = ShiroTokenExtractor.extractToken(fakeRequest); + assertThat(extractedToken, instanceOf(BasicAuthToken.class)); + assertThat(extractedToken.asAuthHeaderValue(), equalTo(basicAuthHeader)); + } + + public void testAuthorizationHeaderExtractionWithUnknownToken() { + String authHeader = "foo"; + RestRequest fakeRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( + Map.of(ShiroTokenExtractor.AUTH_HEADER_NAME, List.of(authHeader)) + ).build(); + AuthToken extractedToken = ShiroTokenExtractor.extractToken(fakeRequest); + assertNull(extractedToken); + } +} diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index fbf90b97d1e8f..3fe0f1dc7cb83 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -579,7 +579,7 @@ public ActionModule( actionPlugins.stream().flatMap(p -> p.indicesAliasesRequestValidators().stream()).collect(Collectors.toList()) ); - restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService, identityService); + restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService); } public Map> getActions() { diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java index 0d6f965c7033f..4f87c01258396 100644 --- a/server/src/main/java/org/opensearch/rest/RestController.java +++ b/server/src/main/java/org/opensearch/rest/RestController.java @@ -41,7 +41,6 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.path.PathTrie; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.Streams; import org.opensearch.common.xcontent.XContentType; @@ -56,11 +55,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.http.HttpChunk; import org.opensearch.http.HttpServerTransport; -import org.opensearch.identity.IdentityService; -import org.opensearch.identity.Subject; -import org.opensearch.identity.UserSubject; -import org.opensearch.identity.tokens.AuthToken; -import org.opensearch.identity.tokens.RestTokenExtractor; import org.opensearch.usage.UsageService; import java.io.ByteArrayOutputStream; @@ -125,25 +119,23 @@ public class RestController implements HttpServerTransport.Dispatcher { /** Rest headers that are copied to internal requests made during a rest request. */ private final Set headersToCopy; private final UsageService usageService; - private final IdentityService identityService; public RestController( Set headersToCopy, UnaryOperator handlerWrapper, NodeClient client, CircuitBreakerService circuitBreakerService, - UsageService usageService, - IdentityService identityService + UsageService usageService ) { this.headersToCopy = headersToCopy; this.usageService = usageService; if (handlerWrapper == null) { handlerWrapper = h -> h; // passthrough if no wrapper set } + this.handlerWrapper = handlerWrapper; this.client = client; this.circuitBreakerService = circuitBreakerService; - this.identityService = identityService; registerHandlerNoWrap( RestRequest.Method.GET, "/favicon.ico", @@ -472,11 +464,6 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel return; } } else { - if (FeatureFlags.isEnabled(FeatureFlags.IDENTITY)) { - if (!handleAuthenticateUser(request, channel)) { - return; - } - } dispatchRequest(request, channel, handler); return; } @@ -587,43 +574,6 @@ private void handleBadRequest(String uri, RestRequest.Method method, RestChannel } } - /** - * Attempts to extract auth token and login. - * - * @return false if there was an error and the request should not continue being dispatched - * */ - private boolean handleAuthenticateUser(final RestRequest request, final RestChannel channel) { - try { - final AuthToken token = RestTokenExtractor.extractToken(request); - // If no token was found, continue executing the request - if (token == null) { - // Authentication did not fail so return true. Authorization is handled at the action level. - return true; - } - final Subject currentSubject = identityService.getCurrentSubject(); - if (currentSubject instanceof UserSubject) { - ((UserSubject) currentSubject).authenticate(token); - logger.debug("Logged in as user " + currentSubject); - } - } catch (final Exception e) { - try { - final BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse( - channel, - RestStatus.UNAUTHORIZED, - e.getMessage() - ); - channel.sendResponse(bytesRestResponse); - } catch (final Exception ex) { - final BytesRestResponse bytesRestResponse = new BytesRestResponse(RestStatus.UNAUTHORIZED, ex.getMessage()); - channel.sendResponse(bytesRestResponse); - } - return false; - } - - // Authentication did not fail so return true. Authorization is handled at the action level. - return true; - } - /** * Get the valid set of HTTP methods for a REST request. */ diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 5ae1bdce48cd5..bf1d52b49cb1f 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -153,8 +153,7 @@ public List> getExtensionSettings() { null, new NodeClient(Settings.EMPTY, threadPool), new NoneCircuitBreakerService(), - new UsageService(), - new IdentityService(Settings.EMPTY, threadPool, List.of()) + new UsageService() ); when(actionModule.getDynamicActionRegistry()).thenReturn(mock(DynamicActionRegistry.class)); when(actionModule.getRestController()).thenReturn(restController); diff --git a/server/src/test/java/org/opensearch/rest/RestControllerTests.java b/server/src/test/java/org/opensearch/rest/RestControllerTests.java index ef9257d746573..f7f1b02847854 100644 --- a/server/src/test/java/org/opensearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/opensearch/rest/RestControllerTests.java @@ -55,13 +55,11 @@ import org.opensearch.http.HttpResponse; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.HttpStats; -import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.rest.action.admin.indices.RestCreateIndexAction; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpNodeClient; import org.opensearch.test.rest.FakeRestRequest; -import org.opensearch.threadpool.ThreadPool; import org.opensearch.usage.UsageService; import org.junit.After; import org.junit.Before; @@ -97,7 +95,6 @@ public class RestControllerTests extends OpenSearchTestCase { private RestController restController; private HierarchyCircuitBreakerService circuitBreakerService; private UsageService usageService; - private IdentityService identityService; private NodeClient client; @Before @@ -115,11 +112,9 @@ public void setup() { // we can do this here only because we know that we don't adjust breaker settings dynamically in the test inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS); - identityService = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), List.of()); - HttpServerTransport httpServerTransport = new TestHttpServerTransport(); client = new NoOpNodeClient(this.getTestName()); - restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService, identityService); + restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService); restController.registerHandler( RestRequest.Method.GET, "/", @@ -140,7 +135,7 @@ public void teardown() throws IOException { } public void testDefaultRestControllerGetAllHandlersContainsFavicon() { - final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService); Iterator handlers = restController.getAllHandlers(); assertTrue(handlers.hasNext()); MethodHandlers faviconHandler = handlers.next(); @@ -150,7 +145,7 @@ public void testDefaultRestControllerGetAllHandlersContainsFavicon() { } public void testRestControllerGetAllHandlers() { - final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService); restController.registerHandler(RestRequest.Method.PATCH, "/foo", mock(RestHandler.class)); restController.registerHandler(RestRequest.Method.GET, "/foo", mock(RestHandler.class)); @@ -175,7 +170,7 @@ public void testApplyRelevantHeaders() throws Exception { Set headers = new HashSet<>( Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", true)) ); - final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService); Map> restHeaders = new HashMap<>(); restHeaders.put("header.1", Collections.singletonList("true")); restHeaders.put("header.2", Collections.singletonList("true")); @@ -211,7 +206,7 @@ public void testRequestWithDisallowedMultiValuedHeader() { Set headers = new HashSet<>( Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", false)) ); - final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService); Map> restHeaders = new HashMap<>(); restHeaders.put("header.1", Collections.singletonList("boo")); restHeaders.put("header.2", Arrays.asList("foo", "bar")); @@ -226,14 +221,7 @@ public void testRequestWithDisallowedMultiValuedHeaderButSameValues() { Set headers = new HashSet<>( Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", false)) ); - final RestController restController = new RestController( - headers, - null, - client, - circuitBreakerService, - usageService, - identityService - ); + final RestController restController = new RestController(headers, null, client, circuitBreakerService, usageService); Map> restHeaders = new HashMap<>(); restHeaders.put("header.1", Collections.singletonList("boo")); restHeaders.put("header.2", Arrays.asList("foo", "foo")); @@ -294,7 +282,7 @@ public void testRegisterWithDeprecatedHandler() { } public void testRegisterSecondMethodWithDifferentNamedWildcard() { - final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService); RestRequest.Method firstMethod = randomFrom(RestRequest.Method.values()); RestRequest.Method secondMethod = randomFrom( @@ -322,7 +310,7 @@ public void testRestHandlerWrapper() throws Exception { final RestController restController = new RestController(Collections.emptySet(), h -> { assertSame(handler, h); return (RestRequest request, RestChannel channel, NodeClient client) -> wrapperCalled.set(true); - }, client, circuitBreakerService, usageService, identityService); + }, client, circuitBreakerService, usageService); restController.registerHandler(RestRequest.Method.GET, "/wrapped", handler); RestRequest request = testRestRequest("/wrapped", "{}", MediaTypeRegistry.JSON); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST); @@ -385,7 +373,7 @@ public void testDispatchRequiresContentTypeForRequestsWithContent() { String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead())); RestRequest request = testRestRequest("/", content, null); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE); - restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService, identityService); + restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService); restController.registerHandler( RestRequest.Method.GET, "/", diff --git a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java index 983121a4f481d..b8602cdc20e6a 100644 --- a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java +++ b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java @@ -39,12 +39,10 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.core.rest.RestStatus; -import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestChannel; import org.opensearch.test.rest.FakeRestRequest; -import org.opensearch.threadpool.ThreadPool; import org.opensearch.usage.UsageService; import java.util.ArrayList; @@ -56,7 +54,6 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; -import static org.mockito.Mockito.mock; public class RestHttpResponseHeadersTests extends OpenSearchTestCase { @@ -106,17 +103,8 @@ public void testUnsupportedMethodResponseHttpHeader() throws Exception { new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) ); - final Settings settings = Settings.EMPTY; UsageService usageService = new UsageService(); - final IdentityService identityService = new IdentityService(settings, mock(ThreadPool.class), List.of()); - RestController restController = new RestController( - Collections.emptySet(), - null, - null, - circuitBreakerService, - usageService, - identityService - ); + RestController restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService); // A basic RestHandler handles requests to the endpoint RestHandler restHandler = new RestHandler() { diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java index c3cf33f4e9034..6aa1d10d71e50 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java @@ -44,7 +44,6 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.search.AbstractSearchTestCase; @@ -61,7 +60,6 @@ import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import static java.util.Collections.emptyMap; @@ -75,15 +73,7 @@ public class RestValidateQueryActionTests extends AbstractSearchTestCase { private static NodeClient client = new NodeClient(Settings.EMPTY, threadPool); private static UsageService usageService = new UsageService(); - private static IdentityService identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); - private static RestController controller = new RestController( - emptySet(), - null, - client, - new NoneCircuitBreakerService(), - usageService, - identityService - ); + private static RestController controller = new RestController(emptySet(), null, client, new NoneCircuitBreakerService(), usageService); private static RestValidateQueryAction action = new RestValidateQueryAction(); /** diff --git a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java index c7a0fe35b0237..fec1699c9ef64 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java @@ -40,25 +40,20 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; -import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskListener; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpNodeClient; -import org.opensearch.threadpool.ThreadPool; import org.opensearch.usage.UsageService; import org.junit.After; import org.junit.Before; import java.util.Collections; -import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; -import static org.mockito.Mockito.mock; - /** * A common base class for Rest*ActionTests. Provides access to a {@link RestController} * that can be used to register individual REST actions, and test request handling. @@ -70,15 +65,7 @@ public abstract class RestActionTestCase extends OpenSearchTestCase { @Before public void setUpController() { verifyingClient = new VerifyingClient(this.getTestName()); - final IdentityService identityService = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), List.of()); - controller = new RestController( - Collections.emptySet(), - null, - verifyingClient, - new NoneCircuitBreakerService(), - new UsageService(), - identityService - ); + controller = new RestController(Collections.emptySet(), null, verifyingClient, new NoneCircuitBreakerService(), new UsageService()); } @After From 19d8df02444204e2098e012e553066489bb9b679 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 19 Sep 2024 17:22:45 -0400 Subject: [PATCH 036/108] Update version check in yml test file (#15999) Signed-off-by: Gao Binlong Co-authored-by: Gao Binlong --- .../rest-api-spec/test/cluster.put_settings/10_basic.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index 0f4a45e4591a3..ce78709f6ed5e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -103,8 +103,8 @@ --- "Test setting search backpressure cancellation settings": - skip: - version: "- 2.99.99" - reason: "Fixed in 3.0.0" + version: "- 2.17.99" + reason: "Fixed in 2.18.0" - do: cluster.put_settings: @@ -181,8 +181,8 @@ --- "Test setting invalid search backpressure cancellation_rate and cancellation_ratio": - skip: - version: "- 2.99.99" - reason: "Fixed in 3.0.0" + version: "- 2.17.99" + reason: "Fixed in 2.18.0" - do: catch: /search_backpressure.search_task.cancellation_rate must be > 0/ From 94222f1dfaa2fd07598141fb9a894ebca881e201 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Thu, 19 Sep 2024 15:08:45 -0700 Subject: [PATCH 037/108] Update big5 workload config to use the ordered documents snapshot (#16005) Signed-off-by: Rishabh Singh --- .github/benchmark-configs.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/benchmark-configs.json b/.github/benchmark-configs.json index 4ada715d21495..27b7228e1203a 100644 --- a/.github/benchmark-configs.json +++ b/.github/benchmark-configs.json @@ -75,7 +75,7 @@ "SINGLE_NODE_CLUSTER": "true", "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -126,7 +126,7 @@ "SINGLE_NODE_CLUSTER": "true", "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -176,7 +176,7 @@ "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", "ADDITIONAL_CONFIG": "search.concurrent_segment_search.enabled:true", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -194,7 +194,7 @@ "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:all", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -212,7 +212,7 @@ "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:auto", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, From b3696116965ab45b675731af04e6defc34602f83 Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:22:48 -0700 Subject: [PATCH 038/108] Add successfulSearchShardIndices in searchRequestContext before onRequestEnd (#15967) Signed-off-by: David Zane Signed-off-by: Ankit Jain Co-authored-by: Ankit Jain --- CHANGELOG.md | 1 + .../action/search/AbstractSearchAsyncAction.java | 3 +++ .../action/search/SearchRequestContext.java | 14 ++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5aef4bb551d3..eebebec672058 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) +- Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) ### Dependencies diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index f0fc05c595d6f..836083f91b995 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -754,6 +754,9 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At } searchRequestContext.setTotalHits(internalSearchResponse.hits().getTotalHits()); searchRequestContext.setShardStats(results.getNumShards(), successfulOps.get(), skippedOps.get(), failures.length); + searchRequestContext.setSuccessfulSearchShardIndices( + results.getSuccessfulResults().map(result -> result.getSearchShardTarget().getIndex()).collect(Collectors.toSet()) + ); onPhaseEnd(searchRequestContext); onRequestEnd(searchRequestContext); listener.onResponse(buildSearchResponse(internalSearchResponse, failures, scrollId, searchContextId)); diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java index 111d9c64550b3..376cf71448d5c 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Supplier; @@ -36,6 +37,7 @@ public class SearchRequestContext { private final Map phaseTookMap; private TotalHits totalHits; private final EnumMap shardStats; + private Set successfulSearchShardIndices; private final SearchRequest searchRequest; private final LinkedBlockingQueue phaseResourceUsage; @@ -141,6 +143,18 @@ public List getPhaseResourceUsage() { public SearchRequest getRequest() { return searchRequest; } + + void setSuccessfulSearchShardIndices(Set successfulSearchShardIndices) { + this.successfulSearchShardIndices = successfulSearchShardIndices; + } + + /** + * @return A {@link List} of {@link String} representing the names of the indices that were + * successfully queried at the shard level. + */ + public Set getSuccessfulSearchShardIndices() { + return successfulSearchShardIndices; + } } enum ShardStatsFieldNames { From 9537d39cc7f4a55c20b7e6f3f3e1a8cec5cb5ec7 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Thu, 19 Sep 2024 18:11:35 -0700 Subject: [PATCH 039/108] add concurrency check to gradle-check workflow (#16007) Signed-off-by: Rishabh Singh --- .github/workflows/gradle-check.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml index 1b9b30625eb83..1421eeb7f7576 100644 --- a/.github/workflows/gradle-check.yml +++ b/.github/workflows/gradle-check.yml @@ -8,6 +8,10 @@ on: pull_request_target: types: [opened, synchronize, reopened] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + permissions: contents: read # to fetch code (actions/checkout) From 2b670cc11f52ada701cbf867ce8ad4527d8b6072 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Fri, 20 Sep 2024 09:36:00 +0530 Subject: [PATCH 040/108] Add support for async deletion in S3BlobContainer (#15621) * Add support for async deletion in S3BlobContainer Signed-off-by: Ashish Singh * Move helper methods to helper class Signed-off-by: Ashish Singh * Minor refactor Signed-off-by: Ashish Singh * Add UTs Signed-off-by: Ashish Singh * Add more tests Signed-off-by: Ashish Singh * Integrate async deletion in the snapshot interactions Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- CHANGELOG.md | 1 + .../s3/S3BlobStoreRepositoryTests.java | 1 + .../s3/S3RepositoryThirdPartyTests.java | 8 + .../repositories/s3/S3BlobContainer.java | 124 +++++ .../repositories/s3/S3BlobStore.java | 2 +- .../repositories/s3/StatsMetricPublisher.java | 4 + .../s3/async/S3AsyncDeleteHelper.java | 95 ++++ .../s3/S3BlobStoreContainerTests.java | 507 +++++++++++++++++- .../s3/async/S3AsyncDeleteHelperTests.java | 236 ++++++++ .../mocks/MockFsAsyncBlobContainer.java | 12 + .../AsyncMultiStreamBlobContainer.java | 5 + ...syncMultiStreamEncryptedBlobContainer.java | 10 + .../common/settings/ClusterSettings.java | 3 + .../blobstore/BlobStoreRepository.java | 60 ++- .../BlobStoreTransferServiceTests.java | 12 + .../snapshots/BlobStoreFormatTests.java | 12 + 16 files changed, 1086 insertions(+), 6 deletions(-) create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index eebebec672058..abcadde83f000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added +- Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java index c5438d58e679d..944de326d144c 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -153,6 +153,7 @@ protected Settings nodeSettings(int nodeOrdinal) { // Disable request throttling because some random values in tests might generate too many failures for the S3 client .put(S3ClientSettings.USE_THROTTLE_RETRIES_SETTING.getConcreteSettingForNamespace("test").getKey(), false) .put(S3ClientSettings.PROXY_TYPE_SETTING.getConcreteSettingForNamespace("test").getKey(), ProxySettings.ProxyType.DIRECT) + .put(BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.getKey(), false) .put(super.nodeSettings(nodeOrdinal)) .setSecureSettings(secureSettings); diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java index 7db9a0d3ba790..f0e40db965646 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -55,6 +55,14 @@ public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase { + @Override + protected Settings nodeSettings() { + return Settings.builder() + .put(super.nodeSettings()) + .put(BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.getKey(), false) + .build(); + } + @Override @Before @SuppressForbidden(reason = "Need to set system property here for AWS SDK v2") diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index b489a3cc85037..1a402e8431e25 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -62,6 +62,7 @@ import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher; import software.amazon.awssdk.utils.CollectionUtils; import org.apache.logging.log4j.LogManager; @@ -90,6 +91,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.repositories.s3.async.S3AsyncDeleteHelper; import org.opensearch.repositories.s3.async.SizeBasedBlockingQ; import org.opensearch.repositories.s3.async.UploadRequest; import org.opensearch.repositories.s3.utils.HttpRangeUtils; @@ -109,6 +111,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + import static org.opensearch.repositories.s3.S3Repository.MAX_FILE_SIZE; import static org.opensearch.repositories.s3.S3Repository.MAX_FILE_SIZE_USING_MULTIPART; import static org.opensearch.repositories.s3.S3Repository.MIN_PART_SIZE_USING_MULTIPART; @@ -875,4 +880,123 @@ CompletableFuture getBlobMetadata(S3AsyncClient s3A return SocketAccess.doPrivileged(() -> s3AsyncClient.getObjectAttributes(getObjectAttributesRequest)); } + + @Override + public void deleteAsync(ActionListener completionListener) { + try (AmazonAsyncS3Reference asyncClientReference = blobStore.asyncClientReference()) { + S3AsyncClient s3AsyncClient = asyncClientReference.get().client(); + + ListObjectsV2Request listRequest = ListObjectsV2Request.builder().bucket(blobStore.bucket()).prefix(keyPath).build(); + ListObjectsV2Publisher listPublisher = s3AsyncClient.listObjectsV2Paginator(listRequest); + + AtomicLong deletedBlobs = new AtomicLong(); + AtomicLong deletedBytes = new AtomicLong(); + + CompletableFuture listingFuture = new CompletableFuture<>(); + + listPublisher.subscribe(new Subscriber<>() { + private Subscription subscription; + private final List objectsToDelete = new ArrayList<>(); + private CompletableFuture deletionChain = CompletableFuture.completedFuture(null); + + @Override + public void onSubscribe(Subscription s) { + this.subscription = s; + subscription.request(1); + } + + @Override + public void onNext(ListObjectsV2Response response) { + response.contents().forEach(s3Object -> { + deletedBlobs.incrementAndGet(); + deletedBytes.addAndGet(s3Object.size()); + objectsToDelete.add(s3Object.key()); + }); + + int bulkDeleteSize = blobStore.getBulkDeletesSize(); + if (objectsToDelete.size() >= bulkDeleteSize) { + int fullBatchesCount = objectsToDelete.size() / bulkDeleteSize; + int itemsToDelete = fullBatchesCount * bulkDeleteSize; + + List batchToDelete = new ArrayList<>(objectsToDelete.subList(0, itemsToDelete)); + objectsToDelete.subList(0, itemsToDelete).clear(); + + deletionChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + batchToDelete, + deletionChain, + () -> subscription.request(1) + ); + } else { + subscription.request(1); + } + } + + @Override + public void onError(Throwable t) { + listingFuture.completeExceptionally(new IOException("Failed to list objects for deletion", t)); + } + + @Override + public void onComplete() { + if (!objectsToDelete.isEmpty()) { + deletionChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + objectsToDelete, + deletionChain, + null + ); + } + deletionChain.whenComplete((v, throwable) -> { + if (throwable != null) { + listingFuture.completeExceptionally(throwable); + } else { + listingFuture.complete(null); + } + }); + } + }); + + listingFuture.whenComplete((v, throwable) -> { + if (throwable != null) { + completionListener.onFailure( + throwable instanceof Exception + ? (Exception) throwable + : new IOException("Unexpected error during async deletion", throwable) + ); + } else { + completionListener.onResponse(new DeleteResult(deletedBlobs.get(), deletedBytes.get())); + } + }); + } catch (Exception e) { + completionListener.onFailure(new IOException("Failed to initiate async deletion", e)); + } + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + if (blobNames.isEmpty()) { + completionListener.onResponse(null); + return; + } + + try (AmazonAsyncS3Reference asyncClientReference = blobStore.asyncClientReference()) { + S3AsyncClient s3AsyncClient = asyncClientReference.get().client(); + + List keysToDelete = blobNames.stream().map(this::buildKey).collect(Collectors.toList()); + + S3AsyncDeleteHelper.executeDeleteChain(s3AsyncClient, blobStore, keysToDelete, CompletableFuture.completedFuture(null), null) + .whenComplete((v, throwable) -> { + if (throwable != null) { + completionListener.onFailure(new IOException("Failed to delete blobs " + blobNames, throwable)); + } else { + completionListener.onResponse(null); + } + }); + } catch (Exception e) { + completionListener.onFailure(new IOException("Failed to initiate async blob deletion", e)); + } + } } diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java index f688be9216b8f..90bfa11e18481 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java @@ -63,7 +63,7 @@ import static org.opensearch.repositories.s3.S3Repository.STORAGE_CLASS_SETTING; import static org.opensearch.repositories.s3.S3Repository.UPLOAD_RETRY_ENABLED; -class S3BlobStore implements BlobStore { +public class S3BlobStore implements BlobStore { private static final Logger logger = LogManager.getLogger(S3BlobStore.class); diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java index 8d2772d42ebca..9f73c67df3b18 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java @@ -95,6 +95,10 @@ public void publish(MetricCollection metricCollection) { public void close() {} }; + public MetricPublisher getDeleteObjectsMetricPublisher() { + return deleteObjectsMetricPublisher; + } + public MetricPublisher getObjectMetricPublisher = new MetricPublisher() { @Override public void publish(MetricCollection metricCollection) { diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java new file mode 100644 index 0000000000000..eed95c0e68ef1 --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.repositories.s3.S3BlobStore; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +public class S3AsyncDeleteHelper { + private static final Logger logger = LogManager.getLogger(S3AsyncDeleteHelper.class); + + public static CompletableFuture executeDeleteChain( + S3AsyncClient s3AsyncClient, + S3BlobStore blobStore, + List objectsToDelete, + CompletableFuture currentChain, + Runnable afterDeleteAction + ) { + List> batches = createDeleteBatches(objectsToDelete, blobStore.getBulkDeletesSize()); + CompletableFuture newChain = currentChain.thenCompose(v -> executeDeleteBatches(s3AsyncClient, blobStore, batches)); + if (afterDeleteAction != null) { + newChain = newChain.thenRun(afterDeleteAction); + } + return newChain; + } + + static List> createDeleteBatches(List keys, int bulkDeleteSize) { + List> batches = new ArrayList<>(); + for (int i = 0; i < keys.size(); i += bulkDeleteSize) { + batches.add(keys.subList(i, Math.min(keys.size(), i + bulkDeleteSize))); + } + return batches; + } + + static CompletableFuture executeDeleteBatches(S3AsyncClient s3AsyncClient, S3BlobStore blobStore, List> batches) { + CompletableFuture allDeletesFuture = CompletableFuture.completedFuture(null); + + for (List batch : batches) { + allDeletesFuture = allDeletesFuture.thenCompose(v -> executeSingleDeleteBatch(s3AsyncClient, blobStore, batch)); + } + + return allDeletesFuture; + } + + static CompletableFuture executeSingleDeleteBatch(S3AsyncClient s3AsyncClient, S3BlobStore blobStore, List batch) { + DeleteObjectsRequest deleteRequest = bulkDelete(blobStore.bucket(), batch, blobStore); + return s3AsyncClient.deleteObjects(deleteRequest).thenApply(S3AsyncDeleteHelper::processDeleteResponse); + } + + static Void processDeleteResponse(DeleteObjectsResponse deleteObjectsResponse) { + if (!deleteObjectsResponse.errors().isEmpty()) { + logger.warn( + () -> new ParameterizedMessage( + "Failed to delete some blobs {}", + deleteObjectsResponse.errors() + .stream() + .map(s3Error -> "[" + s3Error.key() + "][" + s3Error.code() + "][" + s3Error.message() + "]") + .collect(Collectors.toList()) + ) + ); + } + return null; + } + + static DeleteObjectsRequest bulkDelete(String bucket, List blobs, S3BlobStore blobStore) { + return DeleteObjectsRequest.builder() + .bucket(bucket) + .delete( + Delete.builder() + .objects(blobs.stream().map(blob -> ObjectIdentifier.builder().key(blob).build()).collect(Collectors.toList())) + .quiet(true) + .build() + ) + .overrideConfiguration(o -> o.addMetricPublisher(blobStore.getStatsMetricPublisher().getDeleteObjectsMetricPublisher())) + .build(); + } +} diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java index 654d8a72690c4..2cb11541d924f 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java @@ -69,6 +69,7 @@ import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher; import org.opensearch.action.LatchedActionListener; import org.opensearch.common.blobstore.BlobContainer; @@ -99,22 +100,28 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; public class S3BlobStoreContainerTests extends OpenSearchTestCase { @@ -1275,6 +1282,504 @@ public void testTransformResponseToInputStreamContainer() throws Exception { assertEquals(inputStream.available(), inputStreamContainer.getInputStream().available()); } + public void testDeleteAsync() throws Exception { + for (int i = 0; i < 100; i++) { + testDeleteAsync(i + 1); + } + } + + private void testDeleteAsync(int bulkDeleteSize) throws InterruptedException { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(bulkDeleteSize); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final List s3Objects = new ArrayList<>(); + int numObjects = randomIntBetween(20, 100); + long totalSize = 0; + for (int i = 0; i < numObjects; i++) { + long size = randomIntBetween(1, 100); + s3Objects.add(S3Object.builder().key(randomAlphaOfLength(10)).size(size).build()); + totalSize += size; + } + + final List responseList = new ArrayList<>(); + int size = 0; + while (size < numObjects) { + int toAdd = randomIntBetween(10, 20); + int endIndex = Math.min(numObjects, size + toAdd); + responseList.add(ListObjectsV2Response.builder().contents(s3Objects.subList(size, endIndex)).build()); + size = endIndex; + } + int expectedDeletedObjectsCall = numObjects / bulkDeleteSize + (numObjects % bulkDeleteSize > 0 ? 1 : 0); + + final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class); + AtomicInteger counter = new AtomicInteger(); + doAnswer(invocation -> { + Subscriber subscriber = invocation.getArgument(0); + subscriber.onSubscribe(new Subscription() { + @Override + public void request(long n) { + int currentCounter = counter.getAndIncrement(); + if (currentCounter < responseList.size()) { + subscriber.onNext(responseList.get(currentCounter)); + } + if (currentCounter == responseList.size()) { + subscriber.onComplete(); + } + } + + @Override + public void cancel() {} + }); + return null; + }).when(listPublisher).subscribe(ArgumentMatchers.>any()); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference deleteResultRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + deleteResultRef.set(deleteResult); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + logger.error("exception during deleteAsync", e); + fail("Unexpected failure: " + e.getMessage()); + } + }); + + latch.await(); + + DeleteResult deleteResult = deleteResultRef.get(); + assertEquals(numObjects, deleteResult.blobsDeleted()); + assertEquals(totalSize, deleteResult.bytesDeleted()); + + verify(s3AsyncClient, times(1)).listObjectsV2Paginator(any(ListObjectsV2Request.class)); + verify(s3AsyncClient, times(expectedDeletedObjectsCall)).deleteObjects(any(DeleteObjectsRequest.class)); + } + + public void testDeleteAsyncFailure() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + // Simulate a failure in listObjectsV2Paginator + RuntimeException simulatedFailure = new RuntimeException("Simulated failure"); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenThrow(simulatedFailure); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(IOException.class, exceptionRef.get().getClass()); + assertEquals("Failed to initiate async deletion", exceptionRef.get().getMessage()); + assertEquals(simulatedFailure, exceptionRef.get().getCause()); + } + + public void testDeleteAsyncListingError() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class); + doAnswer(invocation -> { + Subscriber subscriber = invocation.getArgument(0); + subscriber.onSubscribe(new Subscription() { + @Override + public void request(long n) { + subscriber.onError(new RuntimeException("Simulated listing error")); + } + + @Override + public void cancel() {} + }); + return null; + }).when(listPublisher).subscribe(ArgumentMatchers.>any()); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(IOException.class, exceptionRef.get().getClass()); + assertEquals("Failed to list objects for deletion", exceptionRef.get().getMessage()); + assertNotNull(exceptionRef.get().getCause()); + assertEquals("Simulated listing error", exceptionRef.get().getCause().getMessage()); + } + + public void testDeleteAsyncDeletionError() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class); + doAnswer(invocation -> { + Subscriber subscriber = invocation.getArgument(0); + subscriber.onSubscribe(new Subscription() { + @Override + public void request(long n) { + subscriber.onNext( + ListObjectsV2Response.builder().contents(S3Object.builder().key("test-key").size(100L).build()).build() + ); + subscriber.onComplete(); + } + + @Override + public void cancel() {} + }); + return null; + }).when(listPublisher).subscribe(ArgumentMatchers.>any()); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher); + + // Simulate a failure in deleteObjects + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(new RuntimeException("Simulated delete error")); + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(failedFuture); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(CompletionException.class, exceptionRef.get().getClass()); + assertEquals("java.lang.RuntimeException: Simulated delete error", exceptionRef.get().getMessage()); + assertNotNull(exceptionRef.get().getCause()); + assertEquals("Simulated delete error", exceptionRef.get().getCause().getMessage()); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExists() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + int bulkDeleteSize = randomIntBetween(1, 10); + when(blobStore.getBulkDeletesSize()).thenReturn(bulkDeleteSize); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final List blobNames = new ArrayList<>(); + int size = randomIntBetween(10, 100); + for (int i = 0; i < size; i++) { + blobNames.add(randomAlphaOfLength(10)); + } + int expectedDeleteCalls = size / bulkDeleteSize + (size % bulkDeleteSize > 0 ? 1 : 0); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNull(exceptionRef.get()); + + ArgumentCaptor deleteRequestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient, times(expectedDeleteCalls)).deleteObjects(deleteRequestCaptor.capture()); + + DeleteObjectsRequest capturedRequest = deleteRequestCaptor.getAllValues().stream().findAny().get(); + assertEquals(bucketName, capturedRequest.bucket()); + int totalBlobsDeleted = deleteRequestCaptor.getAllValues() + .stream() + .map(r -> r.delete().objects().size()) + .reduce(Integer::sum) + .get(); + assertEquals(blobNames.size(), totalBlobsDeleted); + List deletedKeys = deleteRequestCaptor.getAllValues() + .stream() + .flatMap(r -> r.delete().objects().stream()) + .map(ObjectIdentifier::key) + .collect(Collectors.toList()); + assertTrue(deletedKeys.containsAll(blobNames)); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExistsFailure() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + // Simulate a failure in deleteObjects + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(new RuntimeException("Simulated delete failure")); + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(failedFuture); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + List blobNames = Arrays.asList("blob1", "blob2", "blob3"); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(IOException.class, exceptionRef.get().getClass()); + assertEquals("Failed to delete blobs " + blobNames, exceptionRef.get().getMessage()); + assertNotNull(exceptionRef.get().getCause()); + assertEquals("java.lang.RuntimeException: Simulated delete failure", exceptionRef.get().getCause().getMessage()); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExistsWithEmptyList() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + List emptyBlobNames = Collections.emptyList(); + + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean onResponseCalled = new AtomicBoolean(false); + AtomicReference exceptionRef = new AtomicReference<>(); + + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(emptyBlobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + onResponseCalled.set(true); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertTrue("onResponse should have been called", onResponseCalled.get()); + assertNull("No exception should have been thrown", exceptionRef.get()); + + // Verify that no interactions with S3AsyncClient occurred + verifyNoInteractions(s3AsyncClient); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExistsInitializationFailure() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + // Simulate a failure when getting the asyncClientReference + RuntimeException simulatedFailure = new RuntimeException("Simulated initialization failure"); + when(blobStore.asyncClientReference()).thenThrow(simulatedFailure); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + List blobNames = Arrays.asList("blob1", "blob2", "blob3"); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull("An exception should have been thrown", exceptionRef.get()); + assertTrue("Exception should be an IOException", exceptionRef.get() instanceof IOException); + assertEquals("Failed to initiate async blob deletion", exceptionRef.get().getMessage()); + assertEquals(simulatedFailure, exceptionRef.get().getCause()); + } + private void mockObjectResponse(S3AsyncClient s3AsyncClient, String bucketName, String blobName, int objectSize) { final InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(objectSize)); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java new file mode 100644 index 0000000000000..d7f924e05cc70 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java @@ -0,0 +1,236 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.S3Error; + +import org.opensearch.repositories.s3.S3BlobStore; +import org.opensearch.repositories.s3.StatsMetricPublisher; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class S3AsyncDeleteHelperTests extends OpenSearchTestCase { + + @Mock + private S3AsyncClient s3AsyncClient; + + @Mock + private S3BlobStore blobStore; + + @Override + public void setUp() throws Exception { + super.setUp(); + MockitoAnnotations.openMocks(this); + } + + public void testExecuteDeleteChain() { + List objectsToDelete = Arrays.asList("key1", "key2", "key3"); + CompletableFuture currentChain = CompletableFuture.completedFuture(null); + + // Mock the deleteObjects method of S3AsyncClient + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + // Mock the getBulkDeletesSize method of S3BlobStore + when(blobStore.getBulkDeletesSize()).thenReturn(2); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture newChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + objectsToDelete, + currentChain, + null + ); + + // Verify that the newChain is completed without any exceptions + assertNotNull(newChain); + assertTrue(newChain.isDone()); + assertFalse(newChain.isCompletedExceptionally()); + + // Verify that the deleteObjects method of S3AsyncClient was called with the expected request + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient, times(2)).deleteObjects(requestCaptor.capture()); + + List capturedRequests = requestCaptor.getAllValues(); + assertEquals(2, capturedRequests.size()); + + // Verify that the requests have the expected metric publisher added + for (DeleteObjectsRequest request : capturedRequests) { + assertNotNull(request.overrideConfiguration()); + assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher)); + } + } + + public void testCreateDeleteBatches() { + List keys = Arrays.asList("key1", "key2", "key3", "key4", "key5", "key6"); + int bulkDeleteSize = 3; + + List> batches = S3AsyncDeleteHelper.createDeleteBatches(keys, bulkDeleteSize); + + assertEquals(2, batches.size()); + assertEquals(Arrays.asList("key1", "key2", "key3"), batches.get(0)); + assertEquals(Arrays.asList("key4", "key5", "key6"), batches.get(1)); + } + + public void testExecuteSingleDeleteBatch() throws Exception { + List batch = Arrays.asList("key1", "key2"); + DeleteObjectsResponse expectedResponse = DeleteObjectsResponse.builder().build(); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(CompletableFuture.completedFuture(expectedResponse)); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture future = S3AsyncDeleteHelper.executeSingleDeleteBatch(s3AsyncClient, blobStore, batch); + + assertNotNull(future); + assertTrue(future.isDone()); + assertFalse(future.isCompletedExceptionally()); + future.join(); // Wait for the CompletableFuture to complete + + // Verify that the deleteObjects method of S3AsyncClient was called with the expected request + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient).deleteObjects(requestCaptor.capture()); + + DeleteObjectsRequest capturedRequest = requestCaptor.getValue(); + assertEquals(blobStore.bucket(), capturedRequest.bucket()); + assertEquals(batch.size(), capturedRequest.delete().objects().size()); + assertTrue(capturedRequest.delete().objects().stream().map(ObjectIdentifier::key).collect(Collectors.toList()).containsAll(batch)); + } + + public void testProcessDeleteResponse() { + DeleteObjectsResponse response = DeleteObjectsResponse.builder() + .errors( + Arrays.asList( + S3Error.builder().key("key1").code("Code1").message("Message1").build(), + S3Error.builder().key("key2").code("Code2").message("Message2").build() + ) + ) + .build(); + + // Call the processDeleteResponse method + S3AsyncDeleteHelper.processDeleteResponse(response); + } + + public void testBulkDelete() { + List blobs = Arrays.asList("key1", "key2", "key3"); + String bucketName = "my-bucket"; + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + DeleteObjectsRequest request = S3AsyncDeleteHelper.bulkDelete(bucketName, blobs, blobStore); + + assertEquals(bucketName, request.bucket()); + assertEquals(blobs.size(), request.delete().objects().size()); + assertTrue(request.delete().objects().stream().map(ObjectIdentifier::key).collect(Collectors.toList()).containsAll(blobs)); + assertNotNull(request.overrideConfiguration()); + assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher)); + } + + public void testExecuteDeleteBatches() { + List> batches = Arrays.asList(Arrays.asList("key1", "key2"), Arrays.asList("key3", "key4")); + DeleteObjectsResponse expectedResponse = DeleteObjectsResponse.builder().build(); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(CompletableFuture.completedFuture(expectedResponse)); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture future = S3AsyncDeleteHelper.executeDeleteBatches(s3AsyncClient, blobStore, batches); + + assertNotNull(future); + assertTrue(future.isDone()); + assertFalse(future.isCompletedExceptionally()); + future.join(); // Wait for the CompletableFuture to complete + + // Verify that the deleteObjects method of S3AsyncClient was called with the expected requests + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient, times(2)).deleteObjects(requestCaptor.capture()); + + List capturedRequests = requestCaptor.getAllValues(); + assertEquals(2, capturedRequests.size()); + for (DeleteObjectsRequest request : capturedRequests) { + assertNotNull(request.overrideConfiguration()); + assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher)); + } + } + + public void testExecuteDeleteChainWithAfterDeleteAction() { + List objectsToDelete = Arrays.asList("key1", "key2", "key3"); + CompletableFuture currentChain = CompletableFuture.completedFuture(null); + Runnable afterDeleteAction = mock(Runnable.class); + + // Mock the deleteObjects method of S3AsyncClient + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + // Mock the getBulkDeletesSize method of S3BlobStore + when(blobStore.getBulkDeletesSize()).thenReturn(2); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture newChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + objectsToDelete, + currentChain, + afterDeleteAction + ); + + // Verify that the newChain is completed without any exceptions + assertNotNull(newChain); + assertTrue(newChain.isDone()); + assertFalse(newChain.isCompletedExceptionally()); + + // Verify that the afterDeleteAction was called + verify(afterDeleteAction).run(); + } + +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java index d45b4e3deb798..875f11203281a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java @@ -1,3 +1,4 @@ + /* * SPDX-License-Identifier: Apache-2.0 * @@ -12,6 +13,7 @@ import org.opensearch.common.StreamContext; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -146,4 +148,14 @@ public boolean remoteIntegrityCheckSupported() { private boolean isSegmentFile(String filename) { return !filename.endsWith(".tlog") && !filename.endsWith(".ckp"); } + + @Override + public void deleteAsync(ActionListener completionListener) { + throw new UnsupportedOperationException("deleteAsync"); + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + throw new UnsupportedOperationException("deleteBlobsAsyncIgnoringIfNotExists"); + } } diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java index 97f304d776f5c..b769cdc2fe7ab 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java @@ -14,6 +14,7 @@ import org.opensearch.core.action.ActionListener; import java.io.IOException; +import java.util.List; /** * An extension of {@link BlobContainer} that adds {@link AsyncMultiStreamBlobContainer#asyncBlobUpload} to allow @@ -48,4 +49,8 @@ public interface AsyncMultiStreamBlobContainer extends BlobContainer { * by underlying blobContainer. In this case, caller doesn't need to ensure integrity of data. */ boolean remoteIntegrityCheckSupported(); + + void deleteAsync(ActionListener completionListener); + + void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener); } diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java index 82bc7a0baed50..286c01f9dca44 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java @@ -171,4 +171,14 @@ private InputStreamContainer decryptInputStreamContainer(InputStreamContainer in return new InputStreamContainer(decryptedStream, adjustedLength, adjustedPos); } } + + @Override + public void deleteAsync(ActionListener completionListener) { + blobContainer.deleteAsync(completionListener); + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, completionListener); + } } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 09832e2b41b6d..ecdd23530c648 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -779,7 +779,10 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED, RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX, + + // Snapshot related Settings BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING, + BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING, SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index b954560c1bc94..0292cecc36a81 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -50,6 +50,7 @@ import org.opensearch.action.ActionRunnable; import org.opensearch.action.StepListener; import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.RepositoryCleanupInProgress; @@ -69,6 +70,7 @@ import org.opensearch.common.Randomness; import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; @@ -180,6 +182,7 @@ import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -353,6 +356,16 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp Setting.Property.Final ); + /** + * Controls the fixed prefix for the snapshot shard blob path. cluster.snapshot.async-deletion.enable + */ + public static final Setting SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING = Setting.boolSetting( + "cluster.snapshot.async-deletion.enable", + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + protected volatile boolean supportURLRepo; private volatile int maxShardBlobDeleteBatch; @@ -446,6 +459,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp private final String snapshotShardPathPrefix; + private volatile boolean enableAsyncDeletion; + /** * Flag that is set to {@code true} if this instance is started with {@link #metadata} that has a higher value for * {@link RepositoryMetadata#pendingGeneration()} than for {@link RepositoryMetadata#generation()} indicating a full cluster restart @@ -498,6 +513,8 @@ protected BlobStoreRepository( this.recoverySettings = recoverySettings; this.remoteStoreSettings = new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings()); this.snapshotShardPathPrefix = SNAPSHOT_SHARD_PATH_PREFIX_SETTING.get(clusterService.getSettings()); + this.enableAsyncDeletion = SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.get(clusterService.getSettings()); + clusterService.getClusterSettings().addSettingsUpdateConsumer(SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING, this::setEnableAsyncDeletion); } @Override @@ -2082,7 +2099,7 @@ private void executeOneStaleIndexDelete( } // Finally, we delete the [base_path]/indexId folder - deleteResult = deleteResult.add(indexEntry.getValue().delete()); // Deleting the index folder + deleteResult = deleteResult.add(deleteContainer(indexEntry.getValue())); // Deleting the index folder logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexSnId); return deleteResult; } catch (IOException e) { @@ -2115,6 +2132,21 @@ private void executeOneStaleIndexDelete( })); } + private DeleteResult deleteContainer(BlobContainer container) throws IOException { + long startTime = System.nanoTime(); + DeleteResult deleteResult; + if (enableAsyncDeletion && container instanceof AsyncMultiStreamBlobContainer) { + // Use deleteAsync and wait for the result + PlainActionFuture future = new PlainActionFuture<>(); + ((AsyncMultiStreamBlobContainer) container).deleteAsync(future); + deleteResult = future.actionGet(); + } else { + deleteResult = container.delete(); + } + logger.debug(new ParameterizedMessage("[{}] Deleted {} in {}ns", metadata.name(), container.path(), startTime - System.nanoTime())); + return deleteResult; + } + /** * Cleans up the remote store directory if needed. *

This method cleans up segments in the remote store directory for deleted indices. @@ -2318,7 +2350,7 @@ void releaseRemoteStoreLocksAndCleanup( * @return A DeleteResult object representing the result of the deletion operation. * @throws IOException If an I/O error occurs during the deletion process. */ - private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException { + private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException, ExecutionException, InterruptedException { // If the provided ShardInfo is null, return a zero DeleteResult if (shardInfo == null) { return DeleteResult.ZERO; @@ -2330,7 +2362,7 @@ private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException { // Iterate over the shards and delete each shard's data for (int i = 0; i < shardInfo.getShardCount(); i++) { // Call the delete method on the shardContainer and accumulate the result - deleteResult = deleteResult.add(shardContainer(shardInfo.getIndexId(), i).delete()); + deleteResult = deleteResult.add(deleteContainer(shardContainer(shardInfo.getIndexId(), i))); } // Return the accumulated DeleteResult @@ -2714,7 +2746,23 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna private void deleteFromContainer(BlobContainer container, List blobs) throws IOException { logger.trace(() -> new ParameterizedMessage("[{}] Deleting {} from [{}]", metadata.name(), blobs, container.path())); - container.deleteBlobsIgnoringIfNotExists(blobs); + long startTime = System.nanoTime(); + if (enableAsyncDeletion && container instanceof AsyncMultiStreamBlobContainer) { + PlainActionFuture future = new PlainActionFuture<>(); + ((AsyncMultiStreamBlobContainer) container).deleteBlobsAsyncIgnoringIfNotExists(blobs, future); + future.actionGet(); + } else { + container.deleteBlobsIgnoringIfNotExists(blobs); + } + logger.debug( + () -> new ParameterizedMessage( + "[{}] Deletion {} from [{}] took {}ns", + metadata.name(), + blobs, + container.path(), + System.nanoTime() - startTime + ) + ); } private BlobPath indicesPath() { @@ -4565,4 +4613,8 @@ public String toString() { return name; } } + + public void setEnableAsyncDeletion(boolean enableAsyncDeletion) { + this.enableAsyncDeletion = enableAsyncDeletion; + } } diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java index 10e4cc6cfb1ef..fddc6c0c94005 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -51,6 +52,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Base64; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -340,5 +342,15 @@ public boolean remoteIntegrityCheckSupported() { public BlobContainer getDelegate() { return delegate; } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + throw new RuntimeException("deleteBlobsAsyncIgnoringIfNotExists not supported"); + } + + @Override + public void deleteAsync(ActionListener completionListener) { + throw new RuntimeException("deleteAsync not supported"); + } } } diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index 95a8267734a07..72798f7691eb1 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -39,6 +39,7 @@ import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -63,6 +64,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -335,5 +337,15 @@ public boolean remoteIntegrityCheckSupported() { public BlobContainer getDelegate() { return delegate; } + + @Override + public void deleteAsync(ActionListener completionListener) { + throw new RuntimeException("deleteAsync not supported"); + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + throw new RuntimeException("deleteBlobsAsyncIgnoringIfNotExists not supported"); + } } } From f0ea056a93383c4ca7cac361fdf8c73e8afc3941 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 20 Sep 2024 08:42:40 -0400 Subject: [PATCH 041/108] Update protobuf from 3.25.4 to 3.25.5 (#16011) * Update protobuf from 3.25.4 to 3.25.5 Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- server/licenses/protobuf-java-3.25.4.jar.sha1 | 1 - server/licenses/protobuf-java-3.25.5.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 server/licenses/protobuf-java-3.25.4.jar.sha1 create mode 100644 server/licenses/protobuf-java-3.25.5.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index abcadde83f000..32fdfab619b3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) +- Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) ### Changed diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 0c6d0536dd1d5..63817289e80c0 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -20,7 +20,7 @@ woodstox = 6.4.0 kotlin = 1.7.10 antlr4 = 4.13.1 guava = 32.1.1-jre -protobuf = 3.25.4 +protobuf = 3.25.5 jakarta_annotation = 1.3.5 google_http_client = 1.44.1 tdigest = 3.3 diff --git a/server/licenses/protobuf-java-3.25.4.jar.sha1 b/server/licenses/protobuf-java-3.25.4.jar.sha1 deleted file mode 100644 index 21b6fdfd24dc8..0000000000000 --- a/server/licenses/protobuf-java-3.25.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -43fcb86e4a411516c7fc681450f1516de0b862a2 \ No newline at end of file diff --git a/server/licenses/protobuf-java-3.25.5.jar.sha1 b/server/licenses/protobuf-java-3.25.5.jar.sha1 new file mode 100644 index 0000000000000..72b42c9efc85a --- /dev/null +++ b/server/licenses/protobuf-java-3.25.5.jar.sha1 @@ -0,0 +1 @@ +5ae5c9ec39930ae9b5a61b32b93288818ec05ec1 \ No newline at end of file From 4963792c175ab3eb30a493e81360dce95b5b2811 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Fri, 20 Sep 2024 22:24:20 +0800 Subject: [PATCH 042/108] Add more yaml rest tests for search backpressure settings (#16012) Signed-off-by: Gao Binlong --- .../test/cluster.put_settings/10_basic.yml | 142 -------- .../test/search.backpressure/10_basic.yml | 340 ++++++++++++++++++ 2 files changed, 340 insertions(+), 142 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index ce78709f6ed5e..825bac9f91649 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -69,145 +69,3 @@ include_defaults: true - match: {defaults.node.attr.testattr: "test"} - ---- -"Test set search backpressure mode": - - - do: - cluster.put_settings: - body: - persistent: - search_backpressure.mode: "monitor_only" - - - match: {persistent: {search_backpressure: {mode: "monitor_only"}}} - ---- -"Test set invalid search backpressure mode": - - - skip: - version: "- 2.8.99" - reason: "Fixed in 2.9.0" - - - do: - catch: bad_request - cluster.put_settings: - body: - persistent: - search_backpressure.mode: "monitor-only" - - - match: {error.root_cause.0.type: "illegal_argument_exception"} - - match: { error.root_cause.0.reason: "Invalid SearchBackpressureMode: monitor-only" } - - match: { status: 400 } - - ---- -"Test setting search backpressure cancellation settings": - - skip: - version: "- 2.17.99" - reason: "Fixed in 2.18.0" - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_task.cancellation_burst: 11 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_task.cancellation_burst: "11"} - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_task.cancellation_rate: 0.1 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_task.cancellation_rate: "0.1"} - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_task.cancellation_ratio: 0.2 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_task.cancellation_ratio: "0.2"} - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_shard_task.cancellation_burst: 12 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_shard_task.cancellation_burst: "12"} - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_shard_task.cancellation_rate: 0.3 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_shard_task.cancellation_rate: "0.3"} - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_shard_task.cancellation_ratio: 0.4 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_shard_task.cancellation_ratio: "0.4"} - ---- -"Test setting invalid search backpressure cancellation_rate and cancellation_ratio": - - skip: - version: "- 2.17.99" - reason: "Fixed in 2.18.0" - - - do: - catch: /search_backpressure.search_task.cancellation_rate must be > 0/ - cluster.put_settings: - body: - transient: - search_backpressure.search_task.cancellation_rate: 0.0 - - - do: - catch: /search_backpressure.search_task.cancellation_ratio must be > 0/ - cluster.put_settings: - body: - transient: - search_backpressure.search_task.cancellation_ratio: 0.0 - - - do: - catch: /search_backpressure.search_shard_task.cancellation_rate must be > 0/ - cluster.put_settings: - body: - transient: - search_backpressure.search_shard_task.cancellation_rate: 0.0 - - - do: - catch: /search_backpressure.search_shard_task.cancellation_ratio must be > 0/ - cluster.put_settings: - body: - transient: - search_backpressure.search_shard_task.cancellation_ratio: 0.0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml new file mode 100644 index 0000000000000..2862dcce072c4 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml @@ -0,0 +1,340 @@ +--- +"Test set search backpressure mode": + - skip: + version: "- 2.3.99" + reason: "Introduced in 2.4.0" + - do: + cluster.put_settings: + body: + persistent: + search_backpressure.mode: "monitor_only" + + - match: {persistent: {search_backpressure: {mode: "monitor_only"}}} + +--- +"Test set invalid search backpressure mode": + - skip: + version: "- 2.7.99" + reason: "Fixed in 2.8.0" + + - do: + catch: bad_request + cluster.put_settings: + body: + persistent: + search_backpressure.mode: "monitor-only" + + - match: {error.root_cause.0.type: "illegal_argument_exception"} + - match: { error.root_cause.0.reason: "Invalid SearchBackpressureMode: monitor-only" } + - match: { status: 400 } + +--- +"Test setting search backpressure cancellation settings": + - skip: + version: "- 2.17.99" + reason: "Fixed in 2.18.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_burst: 11 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_burst: "11"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_rate: 0.1 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_rate: "0.1"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_ratio: 0.2 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_ratio: "0.2"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_burst: 12 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_burst: "12"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_rate: 0.3 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_rate: "0.3"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_ratio: 0.4 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_ratio: "0.4"} + +--- +"Test setting invalid search backpressure cancellation_rate and cancellation_ratio": + - skip: + version: "- 2.17.99" + reason: "Fixed in 2.18.0" + + - do: + catch: /search_backpressure.search_task.cancellation_rate must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_rate: 0.0 + + - do: + catch: /search_backpressure.search_task.cancellation_ratio must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_ratio: 0.0 + + - do: + catch: /search_backpressure.search_shard_task.cancellation_rate must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_rate: 0.0 + + - do: + catch: /search_backpressure.search_shard_task.cancellation_ratio must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_ratio: 0.0 + +--- +"Test setting search backpressure node settings": + - skip: + version: "- 2.3.99" + reason: "Introduced in 2.4.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.node_duress.num_successive_breaches: 10 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.node_duress.num_successive_breaches: "10"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.node_duress.cpu_threshold: 0.85 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.node_duress.cpu_threshold: "0.85"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.node_duress.heap_threshold: 0.75 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.node_duress.heap_threshold: "0.75"} + + +--- +"Test setting search backpressure search_task settings": + - skip: + version: "- 2.3.99" + reason: "Introduced in 2.4.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.elapsed_time_millis_threshold: 50000 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.elapsed_time_millis_threshold: "50000"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.heap_percent_threshold: 0.01 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.heap_percent_threshold: "0.01"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.total_heap_percent_threshold: 0.08 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.total_heap_percent_threshold: "0.08"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.heap_variance: 3.0 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.heap_variance: "3.0"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.heap_moving_average_window_size: 50 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.heap_moving_average_window_size: "50"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cpu_time_millis_threshold: 40000 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cpu_time_millis_threshold: "40000"} + +--- +"Test setting search backpressure search_shard_task settings": + - skip: + version: "- 2.3.99" + reason: "Introduced in 2.4.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.elapsed_time_millis_threshold: 50000 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.elapsed_time_millis_threshold: "50000"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.heap_percent_threshold: 0.01 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.heap_percent_threshold: "0.01"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.total_heap_percent_threshold: 0.08 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.total_heap_percent_threshold: "0.08"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.heap_variance: 3.0 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.heap_variance: "3.0"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.heap_moving_average_window_size: 50 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.heap_moving_average_window_size: "50"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cpu_time_millis_threshold: 40000 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cpu_time_millis_threshold: "40000"} From 05dab3b7eb54a361af3583a322f0a748d6412836 Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Fri, 20 Sep 2024 15:21:39 -0700 Subject: [PATCH 043/108] Avoid infinite loop in flat_object parsing (#15985) * Avoid infinite loop in flat_object parsing We had logic in flat_object parsing that would: 1. Try parsing a flat object field that is not an object or null. 2. Would see an END_ARRAY token, ignore it, and not advance the parser. Combined, this would create a scenario where passing an array of strings for a flat_object would parse the string values, then loop infinitely on the END_ARRAY token. Signed-off-by: Michael Froh * Remove some unused code and add more tests The removed code does not actually seem to affect the logic. Also, I want to be 100% sure that every call to parseToken is guaranteed to call parser.nextToken() at some point. Signed-off-by: Michael Froh * Remove unused parameter from parseToken Thanks for the reminder, @kkewwei! Signed-off-by: Michael Froh * Add skip for newly-added test The test fails on MixedClusterClientYamlTestSuiteIT because 2.x still has the infinite loop until backport. Signed-off-by: Michael Froh --------- Signed-off-by: Michael Froh --- CHANGELOG.md | 1 + .../test/index/90_flat_object.yml | 48 +++++++++++++++- .../xcontent/JsonToStringXContentParser.java | 37 +++--------- .../index/mapper/FlatObjectFieldMapper.java | 57 +++++++++++-------- 4 files changed, 90 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32fdfab619b3e..ae0b0e0e7a235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) - Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) +- Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml index 83d3d273ebd93..c0fc0090abedf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml @@ -62,7 +62,6 @@ setup: }, "required_matches": 1 } - # Do index refresh - do: indices.refresh: @@ -74,7 +73,52 @@ teardown: - do: indices.delete: index: test - +--- +"Invalid docs": + - skip: + version: "- 2.99.99" + reason: "parsing of these objects would infinite loop prior to 2.18" + # The following documents are invalid. + - do: + catch: /parsing_exception/ + index: + index: test + id: 3 + body: { + "ISBN13": "V12154942123242", + "catalog": [ "Arrays in Action" ], + "required_matches": 1 + } + - do: + catch: /parsing_exception/ + index: + index: test + id: 3 + body: { + "ISBN13": "V12154942123242", + "catalog": "Strings in Action", + "required_matches": 1 + } + - do: + catch: /parsing_exception/ + index: + index: test + id: 3 + body: { + "ISBN13": "V12154942123242", + "catalog": 12345, + "required_matches": 1 + } + - do: + catch: /parsing_exception/ + index: + index: test + id: 3 + body: { + "ISBN13": "V12154942123242", + "catalog": [ 12345 ], + "required_matches": 1 + } --- # Verify that mappings under the catalog field did not expand # and no dynamic fields were created. diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java index 2f60fc8f69f87..95a8d9c9495f2 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java +++ b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java @@ -9,6 +9,7 @@ package org.opensearch.common.xcontent; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.AbstractXContentParser; @@ -73,7 +74,7 @@ public XContentParser parseObject() throws IOException { builder.startObject(); LinkedList path = new LinkedList<>(Collections.singleton(fieldTypeName)); while (currentToken() != Token.END_OBJECT) { - parseToken(path, null); + parseToken(path); } // deduplication the fieldName,valueList,valueAndPathList builder.field(this.fieldTypeName, new HashSet<>(keyList)); @@ -87,14 +88,11 @@ public XContentParser parseObject() throws IOException { /** * @return true if the child object contains no_null value, false otherwise */ - private boolean parseToken(Deque path, String currentFieldName) throws IOException { - if (path.size() == 1 && processNoNestedValue()) { - return true; - } + private boolean parseToken(Deque path) throws IOException { boolean isChildrenValueValid = false; boolean visitFieldName = false; if (this.parser.currentToken() == Token.FIELD_NAME) { - currentFieldName = this.parser.currentName(); + final String currentFieldName = this.parser.currentName(); path.addLast(currentFieldName); // Pushing onto the stack *must* be matched by pop visitFieldName = true; String parts = currentFieldName; @@ -106,23 +104,21 @@ private boolean parseToken(Deque path, String currentFieldName) throws I } this.keyList.add(parts); // parts has no dot, so either it's the original fieldName or it's the last part this.parser.nextToken(); // advance to the value of fieldName - isChildrenValueValid = parseToken(path, currentFieldName); // parse the value for fieldName (which will be an array, an object, - // or a primitive value) + isChildrenValueValid = parseToken(path); // parse the value for fieldName (which will be an array, an object, + // or a primitive value) path.removeLast(); // Here is where we pop fieldName from the stack (since we're done with the value of fieldName) // Note that whichever other branch we just passed through has already ended with nextToken(), so we // don't need to call it. } else if (this.parser.currentToken() == Token.START_ARRAY) { parser.nextToken(); while (this.parser.currentToken() != Token.END_ARRAY) { - isChildrenValueValid |= parseToken(path, currentFieldName); + isChildrenValueValid |= parseToken(path); } this.parser.nextToken(); - } else if (this.parser.currentToken() == Token.END_ARRAY) { - // skip } else if (this.parser.currentToken() == Token.START_OBJECT) { parser.nextToken(); while (this.parser.currentToken() != Token.END_OBJECT) { - isChildrenValueValid |= parseToken(path, currentFieldName); + isChildrenValueValid |= parseToken(path); } this.parser.nextToken(); } else { @@ -148,21 +144,6 @@ public void removeKeyOfNullValue() { this.keyList.remove(keyList.size() - 1); } - private boolean processNoNestedValue() throws IOException { - if (parser.currentToken() == Token.VALUE_NULL) { - return true; - } else if (this.parser.currentToken() == Token.VALUE_STRING - || this.parser.currentToken() == Token.VALUE_NUMBER - || this.parser.currentToken() == Token.VALUE_BOOLEAN) { - String value = this.parser.textOrNull(); - if (value != null) { - this.valueList.add(value); - } - return true; - } - return false; - } - private String parseValue() throws IOException { switch (this.parser.currentToken()) { case VALUE_BOOLEAN: @@ -172,7 +153,7 @@ private String parseValue() throws IOException { return this.parser.textOrNull(); // Handle other token types as needed default: - throw new IOException("Unsupported value token type [" + parser.currentToken() + "]"); + throw new ParsingException(parser.getTokenLocation(), "Unexpected value token type [" + parser.currentToken() + "]"); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index bf8f83e1b95df..738efcfafdca1 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -30,6 +30,7 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.xcontent.JsonToStringXContentParser; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; @@ -568,31 +569,41 @@ protected void parseCreateField(ParseContext context) throws IOException { if (context.externalValueSet()) { String value = context.externalValue().toString(); parseValueAddFields(context, value, fieldType().name()); - } else if (context.parser().currentToken() != XContentParser.Token.VALUE_NULL) { - JsonToStringXContentParser jsonToStringParser = new JsonToStringXContentParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.IGNORE_DEPRECATIONS, - context.parser(), - fieldType().name() - ); - /* - JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser - It reads the JSON object and parsed to a list of string - */ - XContentParser parser = jsonToStringParser.parseObject(); - - XContentParser.Token currentToken; - while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - switch (currentToken) { - case FIELD_NAME: - fieldName = parser.currentName(); - break; - case VALUE_STRING: - String value = parser.textOrNull(); - parseValueAddFields(context, value, fieldName); - break; + } else { + XContentParser ctxParser = context.parser(); + if (ctxParser.currentToken() != XContentParser.Token.VALUE_NULL) { + if (ctxParser.currentToken() != XContentParser.Token.START_OBJECT) { + throw new ParsingException( + ctxParser.getTokenLocation(), + "[" + this.name() + "] unexpected token [" + ctxParser.currentToken() + "] in flat_object field value" + ); } + JsonToStringXContentParser jsonToStringParser = new JsonToStringXContentParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.IGNORE_DEPRECATIONS, + ctxParser, + fieldType().name() + ); + /* + JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser + It reads the JSON object and parsed to a list of string + */ + XContentParser parser = jsonToStringParser.parseObject(); + + XContentParser.Token currentToken; + while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + switch (currentToken) { + case FIELD_NAME: + fieldName = parser.currentName(); + break; + case VALUE_STRING: + String value = parser.textOrNull(); + parseValueAddFields(context, value, fieldName); + break; + } + + } } } From 036f6bc836442a35e5f1cab47345e3209e04d085 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Sat, 21 Sep 2024 09:43:27 +0530 Subject: [PATCH 044/108] Add disallow settings update during repository in use ITs (#16001) Signed-off-by: Ashish Singh --- .../snapshots/ConcurrentSnapshotsIT.java | 55 +++++++++++++++++++ .../AbstractSnapshotIntegTestCase.java | 10 ++++ 2 files changed, 65 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java index 15e92f6f7204b..26f8ab5dd8d09 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java @@ -50,6 +50,7 @@ import org.opensearch.common.util.concurrent.UncategorizedExecutionException; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; import org.opensearch.discovery.AbstractDisruptionTestCase; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoryData; @@ -135,6 +136,60 @@ public void testLongRunningSnapshotAllowsConcurrentSnapshot() throws Exception { assertSuccessful(createSlowFuture); } + public void testSettingsUpdateFailWhenCreateSnapshotInProgress() throws Exception { + // Start a cluster with a cluster manager node and a data node + internalCluster().startClusterManagerOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + // Create a repository with random settings + Settings.Builder settings = randomRepositorySettings(); + createRepository(repoName, "mock", settings); + createIndexWithContent("index"); + // Start a full snapshot and block it on the data node + final ActionFuture createSlowFuture = startFullSnapshotBlockedOnDataNode( + "slow-snapshot", + repoName, + dataNode + ); + Thread.sleep(1000); // Wait for the snapshot to start + assertFalse(createSlowFuture.isDone()); // Ensure the snapshot is still in progress + // Attempt to update the repository settings while the snapshot is in progress + IllegalStateException ex = assertThrows(IllegalStateException.class, () -> updateRepository(repoName, "mock", settings)); + // Verify that the update fails with an appropriate exception + assertEquals("trying to modify or unregister repository that is currently used", ex.getMessage()); + unblockNode(repoName, dataNode); // Unblock the snapshot + assertSuccessful(createSlowFuture); // Ensure the snapshot completes successfully + } + + public void testSettingsUpdateFailWhenDeleteSnapshotInProgress() throws InterruptedException { + // Start a cluster with a cluster manager node and a data node + String clusterManagerName = internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + // Create a repository with random settings + Settings.Builder settings = randomRepositorySettings(); + createRepository(repoName, "mock", settings); + createIndexWithContent("index"); + final String snapshotName = "snapshot-1"; + // Create a full snapshot + SnapshotInfo snapshotInfo = createFullSnapshot(repoName, snapshotName); + assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); // Ensure the snapshot was successful + assertEquals(RestStatus.OK, snapshotInfo.status()); // Ensure the snapshot status is OK + // Start deleting the snapshot and block it on the cluster manager node + ActionFuture future = deleteSnapshotBlockedOnClusterManager(repoName, snapshotName); + Thread.sleep(1000); // Wait for the delete operation to start + assertFalse(future.isDone()); // Ensure the delete operation is still in progress + // Attempt to update the repository settings while the delete operation is in progress + IllegalStateException ex = assertThrows( + IllegalStateException.class, + () -> updateRepository(repoName, "mock", randomRepositorySettings()) + ); + // Verify that the update fails with an appropriate exception + assertEquals("trying to modify or unregister repository that is currently used", ex.getMessage()); + unblockNode(repoName, clusterManagerName); // Unblock the delete operation + assertAcked(future.actionGet()); // Wait for the delete operation to complete + } + public void testDeletesAreBatched() throws Exception { internalCluster().startClusterManagerOnlyNode(); final String dataNode = internalCluster().startDataOnlyNode(); diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index a8bb10fe20752..0bfa70a771f65 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -665,6 +665,16 @@ public void onTimeout(TimeValue timeout) { } } + protected ActionFuture deleteSnapshotBlockedOnClusterManager(String repoName, String snapshotName) { + blockClusterManagerFromDeletingIndexNFile(repoName); + return deleteSnapshot(repoName, snapshotName); + } + + protected ActionFuture deleteSnapshot(String repoName, String snapshotName) { + logger.info("--> Deleting snapshot [{}] to repo [{}]", snapshotName, repoName); + return clusterAdmin().prepareDeleteSnapshot(repoName, snapshotName).execute(); + } + protected ActionFuture startFullSnapshotBlockedOnDataNode(String snapshotName, String repoName, String dataNode) throws InterruptedException { blockDataNode(repoName, dataNode); From 13433670227eb6037cc5b743a78099864da9f61c Mon Sep 17 00:00:00 2001 From: Shubh Sahu Date: Mon, 23 Sep 2024 12:38:39 +0530 Subject: [PATCH 045/108] Fixing flaky test testRemoteIndexPathFileExistsAfterMigration (#15977) Signed-off-by: Shubh Sahu --- .../RemoteMigrationIndexMetadataUpdateIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java index b55219e1cb37f..35689deeb8134 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java @@ -19,7 +19,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.util.FileSystemUtils; import org.opensearch.index.remote.RemoteIndexPath; -import org.opensearch.index.remote.RemoteIndexPathUploader; import org.opensearch.index.remote.RemoteStoreEnums; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.InternalTestCluster; @@ -533,17 +532,18 @@ public void testRemoteIndexPathFileExistsAfterMigration() throws Exception { // validate remote index path file exists logger.info("---> Asserting remote index path file exists"); - String fileNamePrefix = String.join(RemoteIndexPathUploader.DELIMITER, indexUUID, "7", RemoteIndexPath.DEFAULT_VERSION); assertTrue(FileSystemUtils.exists(translogRepoPath.resolve(RemoteIndexPath.DIR))); Path[] files = FileSystemUtils.files(translogRepoPath.resolve(RemoteIndexPath.DIR)); assertEquals(1, files.length); - assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(fileNamePrefix))); + logger.info(files[0].toString()); + assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(indexUUID))); assertTrue(FileSystemUtils.exists(segmentRepoPath.resolve(RemoteIndexPath.DIR))); files = FileSystemUtils.files(segmentRepoPath.resolve(RemoteIndexPath.DIR)); assertEquals(1, files.length); - assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(fileNamePrefix))); + logger.info(files[0].toString()); + assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(indexUUID))); } /** From b984b9f56f9b0f9dbb5322bddf04f3b45940b730 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 08:54:47 -0500 Subject: [PATCH 046/108] Bump actions/github-script from 5 to 7 (#16039) * Bump actions/github-script from 5 to 7 Bumps [actions/github-script](https://github.com/actions/github-script) from 5 to 7. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v5...v7) --- updated-dependencies: - dependency-name: actions/github-script dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/delete_backport_branch.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index 91626a0bc5092..1c73d3d250ce3 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -12,7 +12,7 @@ jobs: if: startsWith(github.event.pull_request.head.ref,'backport/') steps: - name: Delete merged branch - uses: actions/github-script@v5 + uses: actions/github-script@v7 with: script: | github.rest.git.deleteRef({ diff --git a/CHANGELOG.md b/CHANGELOG.md index ae0b0e0e7a235..69a054acd24d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) +- Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) ### Changed From c85ee6898b4e3e2f4a2037e42337a7eb8033de9e Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Tue, 24 Sep 2024 07:29:17 +0800 Subject: [PATCH 047/108] Fix search_as_you_type not supporting multi-fields (#15988) * Fix search_as_you_type not supporting multi-fields Signed-off-by: Gao Binlong * Modify change log Signed-off-by: Gao Binlong * Fix test failure Signed-off-by: Gao Binlong * Add more yaml test Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 1 + .../mapper/SearchAsYouTypeFieldMapper.java | 13 +++- .../SearchAsYouTypeFieldMapperTests.java | 14 ++++ ...filtering.yml => 380_bitmap_filtering.yml} | 0 .../test/search/390_search_as_you_type.yml | 75 +++++++++++++++++++ 5 files changed, 101 insertions(+), 2 deletions(-) rename rest-api-spec/src/main/resources/rest-api-spec/test/search/{370_bitmap_filtering.yml => 380_bitmap_filtering.yml} (100%) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 69a054acd24d2..c1041cdf4f64d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) - Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) +- Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988)) - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java index 366e848416328..f08815ebbbd1e 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -264,7 +264,15 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { } ft.setPrefixField(prefixFieldType); ft.setShingleFields(shingleFieldTypes); - return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), prefixFieldMapper, shingleFieldMappers, this); + return new SearchAsYouTypeFieldMapper( + name, + ft, + multiFieldsBuilder.build(this, context), + copyTo.build(), + prefixFieldMapper, + shingleFieldMappers, + this + ); } } @@ -623,12 +631,13 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew public SearchAsYouTypeFieldMapper( String simpleName, SearchAsYouTypeFieldType mappedFieldType, + MultiFields multiFields, CopyTo copyTo, PrefixFieldMapper prefixField, ShingleFieldMapper[] shingleFields, Builder builder ) { - super(simpleName, mappedFieldType, MultiFields.empty(), copyTo); + super(simpleName, mappedFieldType, multiFields, copyTo); this.prefixField = prefixField; this.shingleFields = shingleFields; this.maxShingleSize = builder.maxShingleSize.getValue(); diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index f55ad2e9d659c..7746cb714a019 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -298,6 +298,20 @@ private void assertMultiField(int shingleSize) throws IOException { } } + public void testSubField() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "search_as_you_type") + .startObject("fields") + .startObject("subField") + .field("type", "keyword") + .endObject() + .endObject() + ) + ); + assertThat(mapperService.fieldType("field.subField"), instanceOf(KeywordFieldMapper.KeywordFieldType.class)); + } + public void testIndexOptions() throws IOException { DocumentMapper mapper = createDocumentMapper( fieldMapping(b -> b.field("type", "search_as_you_type").field("index_options", "offsets")) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml rename to rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml new file mode 100644 index 0000000000000..689b3edfd1066 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml @@ -0,0 +1,75 @@ +setup: + - do: + indices.create: + index: test_1 + body: + mappings: + properties: + text: + type: search_as_you_type + fields: + subField: + type: keyword + - do: + index: + index: test_1 + id: 1 + body: { text: test search as you type } + + - do: + indices.refresh: + index: [test_1] + +--- +teardown: + - do: + indices.delete: + index: test_1 + +# related issue: https://github.com/opensearch-project/OpenSearch/issues/5035 +--- +"Test search_as_you_type data type supports multi-fields": + - skip: + version: " - 2.99.99" + reason: "the bug was fixed since 3.0.0" + + - do: + indices.get_mapping: { + index: test_1 + } + + - match: {test_1.mappings.properties.text.type: search_as_you_type} + - match: {test_1.mappings.properties.text.fields.subField.type: keyword} + + - do: + search: + index: test_1 + body: + query: + multi_match: + query: "test search" + type: "bool_prefix" + + - match: {hits.total.value: 1} + + - do: + search: + index: test_1 + body: + query: + multi_match: + query: "test search" + type: "bool_prefix" + fields: ["text.subField"] + + - match: {hits.total.value: 1} + + - do: + search: + index: test_1 + body: + query: + term: + text.subField: "test search as you type" + + - match: {hits.total.value: 1} From eeb2f3997bb84f33f13b848e125051ecf2c2a1c7 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Tue, 24 Sep 2024 11:07:30 +0530 Subject: [PATCH 048/108] Fix flaky test in RemoteStoreMigrationSettingsUpdateIT (#16048) Signed-off-by: Ashish Singh --- .../RemoteStoreMigrationSettingsUpdateIT.java | 7 +++++-- .../java/org/opensearch/test/OpenSearchIntegTestCase.java | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java index 30c597e405f4e..d9e72dd137182 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java @@ -12,6 +12,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.common.unit.TimeValue; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; @@ -68,7 +69,6 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() assertRemoteStoreBackedIndex(indexName2); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15793") public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() throws Exception { logger.info("Initialize cluster: gives non remote cluster manager"); initializeCluster(false); @@ -76,7 +76,10 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix logger.info("Add remote and non-remote nodes"); setClusterMode(MIXED.mode); addRemote = false; - String nonRemoteNodeName = internalCluster().startNode(); + Settings settings = Settings.builder() + .put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), snapshotShardPathFixedPrefix ? "c" : "") + .build(); + String nonRemoteNodeName = internalCluster().startNode(settings); addRemote = true; String remoteNodeName = internalCluster().startNode(); internalCluster().validateClusterFormed(); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index e474ef202b235..68a2b8086a92e 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -403,7 +403,7 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase { private static Boolean segmentsPathFixedPrefix; - private static Boolean snapshotShardPathFixedPrefix; + protected static Boolean snapshotShardPathFixedPrefix; private Path remoteStoreRepositoryPath; @@ -2904,7 +2904,7 @@ private static Settings buildRemoteStoreNodeAttributes( settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean()); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : ""); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : ""); - settings.put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), segmentsPathFixedPrefix ? "c" : ""); + settings.put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), snapshotShardPathFixedPrefix ? "c" : ""); return settings.build(); } From d6bda7dca75dc4d328b61abcd21b35a6bdfcab05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:08:24 +0800 Subject: [PATCH 049/108] Bump dnsjava:dnsjava from 3.6.1 to 3.6.2 in /test/fixtures/hdfs-fixture (#16041) * Bump dnsjava:dnsjava from 3.6.1 to 3.6.2 in /test/fixtures/hdfs-fixture Bumps [dnsjava:dnsjava](https://github.com/dnsjava/dnsjava) from 3.6.1 to 3.6.2. - [Release notes](https://github.com/dnsjava/dnsjava/releases) - [Changelog](https://github.com/dnsjava/dnsjava/blob/master/Changelog) - [Commits](https://github.com/dnsjava/dnsjava/compare/v3.6.1...v3.6.2) --- updated-dependencies: - dependency-name: dnsjava:dnsjava dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1041cdf4f64d..8a32f10ebc691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) +- Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) ### Changed diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 20fd1058a181d..21fe28bfd835d 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -55,7 +55,7 @@ dependencies { exclude group: 'com.nimbusds' exclude module: "commons-configuration2" } - api "dnsjava:dnsjava:3.6.1" + api "dnsjava:dnsjava:3.6.2" api "org.codehaus.jettison:jettison:${versions.jettison}" api "org.apache.commons:commons-compress:${versions.commonscompress}" api "commons-codec:commons-codec:${versions.commonscodec}" From 848e5c676c780d11be78784da87359ae77f47f50 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Tue, 24 Sep 2024 22:35:30 +0800 Subject: [PATCH 050/108] Update version check in yaml test file for the bug fix of avoid infinite loop in flat_object parsing (#16057) Signed-off-by: Gao Binlong --- .../main/resources/rest-api-spec/test/index/90_flat_object.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml index c0fc0090abedf..e8da81d7bee41 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml @@ -76,7 +76,7 @@ teardown: --- "Invalid docs": - skip: - version: "- 2.99.99" + version: "- 2.17.99" reason: "parsing of these objects would infinite loop prior to 2.18" # The following documents are invalid. - do: From 6a2911962539c6288a7f126d529e2393c792fdeb Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 24 Sep 2024 12:49:31 -0400 Subject: [PATCH 051/108] Add support for docker compose v2 in TestFixturesPlugin (#16049) * Update references for docker-compose to docker compose Signed-off-by: Craig Perkins * Support both Signed-off-by: Craig Perkins * Check if V2 is available Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Update CHANGELOG text Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + .../gradle/docker/DockerSupportService.java | 11 +++++++++++ .../gradle/testfixtures/TestFixturesPlugin.java | 9 +++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a32f10ebc691..dfec81c070bab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) ### Changed +- Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) ### Deprecated diff --git a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java index fc78792bb3551..77d7997d6d48d 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java @@ -106,6 +106,7 @@ public DockerAvailability getDockerAvailability() { Version version = null; boolean isVersionHighEnough = false; boolean isComposeAvailable = false; + boolean isComposeV2Available = false; // Check if the Docker binary exists final Optional dockerBinary = getDockerPath(); @@ -129,6 +130,8 @@ public DockerAvailability getDockerAvailability() { if (lastResult.isSuccess() && composePath.isPresent()) { isComposeAvailable = runCommand(composePath.get(), "version").isSuccess(); } + + isComposeV2Available = runCommand(dockerPath, "compose", "version").isSuccess(); } } } @@ -138,6 +141,7 @@ public DockerAvailability getDockerAvailability() { this.dockerAvailability = new DockerAvailability( isAvailable, isComposeAvailable, + isComposeV2Available, isVersionHighEnough, dockerPath, version, @@ -356,6 +360,11 @@ public static class DockerAvailability { */ public final boolean isComposeAvailable; + /** + * True if docker compose is available. + */ + public final boolean isComposeV2Available; + /** * True if the installed Docker version is >= 17.05 */ @@ -379,6 +388,7 @@ public static class DockerAvailability { DockerAvailability( boolean isAvailable, boolean isComposeAvailable, + boolean isComposeV2Available, boolean isVersionHighEnough, String path, Version version, @@ -386,6 +396,7 @@ public static class DockerAvailability { ) { this.isAvailable = isAvailable; this.isComposeAvailable = isComposeAvailable; + this.isComposeV2Available = isComposeV2Available; this.isVersionHighEnough = isVersionHighEnough; this.path = path; this.version = version; diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java index e8772522b19a4..f65e231cd2e50 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java @@ -171,7 +171,11 @@ public void execute(Task task) { .findFirst(); composeExtension.getExecutable().set(dockerCompose.isPresent() ? dockerCompose.get() : "/usr/bin/docker"); - composeExtension.getUseDockerComposeV2().set(false); + if (dockerSupport.get().getDockerAvailability().isComposeV2Available) { + composeExtension.getUseDockerComposeV2().set(true); + } else if (dockerSupport.get().getDockerAvailability().isComposeAvailable) { + composeExtension.getUseDockerComposeV2().set(false); + } tasks.named("composeUp").configure(t -> { // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions @@ -228,7 +232,8 @@ private void maybeSkipTask(Provider dockerSupport, TaskPro private void maybeSkipTask(Provider dockerSupport, Task task) { task.onlyIf(spec -> { - boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeAvailable; + boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeV2Available + || dockerSupport.get().getDockerAvailability().isComposeAvailable; if (isComposeAvailable == false) { LOGGER.info("Task {} requires docker-compose but it is unavailable. Task will be skipped.", task.getPath()); } From b3cc802a11a5685bd32652caecca7e396b00e021 Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Wed, 25 Sep 2024 02:24:00 +0530 Subject: [PATCH 052/108] Change the version to 2.18 as term-check fallback is merged to 2.x (#16062) Signed-off-by: Rajiv Kumar Vaidyanathan --- .../support/clustermanager/term/GetTermVersionResponse.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java index cf8e6085d4b32..f3fd756c6db4b 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java @@ -41,7 +41,7 @@ public GetTermVersionResponse(ClusterStateTermVersion clusterStateTermVersion, b public GetTermVersionResponse(StreamInput in) throws IOException { super(in); this.clusterStateTermVersion = new ClusterStateTermVersion(in); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { this.isStatePresentInRemote = in.readOptionalBoolean(); } else { this.isStatePresentInRemote = false; @@ -51,7 +51,7 @@ public GetTermVersionResponse(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { clusterStateTermVersion.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeOptionalBoolean(isStatePresentInRemote); } } From 12dadcf2aff9ce7e8f1bf4d6afe022684a7d6496 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Wed, 25 Sep 2024 08:35:11 +0530 Subject: [PATCH 053/108] Retry remote state download while bootstrap (#15950) * Retry remote state download while bootstrap Signed-off-by: Sooraj Sinha --- .../RemoteStoreClusterStateRestoreIT.java | 4 +- .../opensearch/gateway/GatewayMetaState.java | 60 +++++++++++-- .../GatewayMetaStatePersistedStateTests.java | 84 ++++++++++++++++++- 3 files changed, 134 insertions(+), 14 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index d078ba05faa12..6a5adf5ea4fb7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -42,6 +42,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.Before; +import java.io.IOError; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -339,10 +340,11 @@ public void testFullClusterRestoreManifestFilePointsToInvalidIndexMetadataPathTh for (UploadedIndexMetadata md : manifest.getIndices()) { Files.move(segmentRepoPath.resolve(md.getUploadedFilename()), segmentRepoPath.resolve("cluster-state/")); } + internalCluster().stopAllNodes(); } catch (IOException e) { throw new RuntimeException(e); } - assertThrows(IllegalStateException.class, () -> addNewNodes(dataNodeCount, clusterManagerNodeCount)); + assertThrows(IOError.class, () -> internalCluster().client()); // Test is complete // Starting a node without remote state to ensure test cleanup diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index b3836edcd7d6c..f5da6df2689bd 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -66,13 +66,13 @@ import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.gateway.remote.model.RemoteClusterStateManifestInfo; import org.opensearch.index.recovery.RemoteStoreRestoreService; -import org.opensearch.index.recovery.RemoteStoreRestoreService.RemoteRestoreResult; import org.opensearch.node.Node; import org.opensearch.plugins.MetadataUpgrader; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.io.Closeable; +import java.io.IOError; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; @@ -109,6 +109,8 @@ public class GatewayMetaState implements Closeable { */ public static final String STALE_STATE_CONFIG_NODE_ID = "STALE_STATE_CONFIG"; + private final Logger logger = LogManager.getLogger(GatewayMetaState.class); + private PersistedStateRegistry persistedStateRegistry; public PersistedState getPersistedState() { @@ -175,15 +177,11 @@ public void start( ); if (ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID) == false) { // Load state from remote - final RemoteRestoreResult remoteRestoreResult = remoteStoreRestoreService.restore( - // Remote Metadata should always override local disk Metadata - // if local disk Metadata's cluster uuid is UNKNOWN_UUID - ClusterState.builder(clusterState).metadata(Metadata.EMPTY_METADATA).build(), - lastKnownClusterUUID, - false, - new String[] {} + clusterState = restoreClusterStateWithRetries( + remoteStoreRestoreService, + clusterState, + lastKnownClusterUUID ); - clusterState = remoteRestoreResult.getClusterState(); } } remotePersistedState = new RemotePersistedState(remoteClusterStateService, lastKnownClusterUUID); @@ -258,6 +256,50 @@ public void start( } } + private ClusterState restoreClusterStateWithRetries( + RemoteStoreRestoreService remoteStoreRestoreService, + ClusterState clusterState, + String lastKnownClusterUUID + ) { + int maxAttempts = 5; + int delayInMills = 200; + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + logger.info("Attempt {} to restore cluster state", attempt); + return restoreClusterState(remoteStoreRestoreService, clusterState, lastKnownClusterUUID); + } catch (Exception e) { + if (attempt == maxAttempts) { + // Throw an Error so that the process is halted. + throw new IOError(e); + } + try { + TimeUnit.MILLISECONDS.sleep(delayInMills); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); // Restore interrupted status + throw new RuntimeException(ie); + } + delayInMills = delayInMills * 2; + } + } + // This statement will never be reached. + return null; + } + + ClusterState restoreClusterState( + RemoteStoreRestoreService remoteStoreRestoreService, + ClusterState clusterState, + String lastKnownClusterUUID + ) { + return remoteStoreRestoreService.restore( + // Remote Metadata should always override local disk Metadata + // if local disk Metadata's cluster uuid is UNKNOWN_UUID + ClusterState.builder(clusterState).metadata(Metadata.EMPTY_METADATA).build(), + lastKnownClusterUUID, + false, + new String[] {} + ).getClusterState(); + } + // exposed so it can be overridden by tests ClusterState prepareInitialClusterState(TransportService transportService, ClusterService clusterService, ClusterState clusterState) { assert clusterState.nodes().getLocalNode() == null : "prepareInitialClusterState must only be called once"; diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index 5ea5241762753..efdb3076f419c 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -1244,14 +1244,72 @@ public void testGatewayForRemoteStateForInitialBootstrapBlocksApplied() throws I } } - private MockGatewayMetaState newGatewayForRemoteState( + public void testGatewayMetaStateRemoteStateDownloadRetries() throws IOException { + MockGatewayMetaState gateway = null; + MockGatewayMetaState gatewayMetaStateSpy = null; + try { + RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class); + when(remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster")).thenReturn("test-cluster-uuid"); + RemoteStoreRestoreService remoteStoreRestoreService = mock(RemoteStoreRestoreService.class); + when(remoteStoreRestoreService.restore(any(), any(), anyBoolean(), any())).thenThrow( + new IllegalStateException("unable to download cluster state") + ).thenReturn(RemoteRestoreResult.build("test-cluster-uuid", null, ClusterState.EMPTY_STATE)); + final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); + gateway = initializeGatewayForRemoteState(true); + gatewayMetaStateSpy = Mockito.spy(gateway); + startGatewayForRemoteState( + gatewayMetaStateSpy, + remoteClusterStateService, + remoteStoreRestoreService, + persistedStateRegistry, + ClusterState.EMPTY_STATE + ); + verify(gatewayMetaStateSpy, times(2)).restoreClusterState(Mockito.any(), Mockito.any(), Mockito.any()); + } finally { + IOUtils.close(gatewayMetaStateSpy); + } + } + + public void testGatewayMetaStateRemoteStateDownloadFailure() throws IOException { + MockGatewayMetaState gateway = null; + final MockGatewayMetaState gatewayMetaStateSpy; + try { + RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class); + when(remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster")).thenReturn("test-cluster-uuid"); + RemoteStoreRestoreService remoteStoreRestoreService = mock(RemoteStoreRestoreService.class); + when(remoteStoreRestoreService.restore(any(), any(), anyBoolean(), any())).thenThrow( + new IllegalStateException("unable to download cluster state") + ); + final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); + gateway = initializeGatewayForRemoteState(true); + gatewayMetaStateSpy = Mockito.spy(gateway); + assertThrows( + Error.class, + () -> startGatewayForRemoteState( + gatewayMetaStateSpy, + remoteClusterStateService, + remoteStoreRestoreService, + persistedStateRegistry, + ClusterState.EMPTY_STATE + ) + ); + verify(gatewayMetaStateSpy, times(5)).restoreClusterState(Mockito.any(), Mockito.any(), Mockito.any()); + } finally { + IOUtils.close(gateway); + } + } + + private MockGatewayMetaState initializeGatewayForRemoteState(boolean prepareFullState) { + return new MockGatewayMetaState(localNode, bigArrays, prepareFullState); + } + + private MockGatewayMetaState startGatewayForRemoteState( + MockGatewayMetaState gateway, RemoteClusterStateService remoteClusterStateService, RemoteStoreRestoreService remoteStoreRestoreService, PersistedStateRegistry persistedStateRegistry, - ClusterState currentState, - boolean prepareFullState + ClusterState currentState ) throws IOException { - MockGatewayMetaState gateway = new MockGatewayMetaState(localNode, bigArrays, prepareFullState); String randomRepoName = "randomRepoName"; String stateRepoTypeAttributeKey = String.format( Locale.getDefault(), @@ -1305,6 +1363,24 @@ private MockGatewayMetaState newGatewayForRemoteState( return gateway; } + private MockGatewayMetaState newGatewayForRemoteState( + RemoteClusterStateService remoteClusterStateService, + RemoteStoreRestoreService remoteStoreRestoreService, + PersistedStateRegistry persistedStateRegistry, + ClusterState currentState, + boolean prepareFullState + ) throws IOException { + MockGatewayMetaState gatewayMetaState = initializeGatewayForRemoteState(prepareFullState); + startGatewayForRemoteState( + gatewayMetaState, + remoteClusterStateService, + remoteStoreRestoreService, + persistedStateRegistry, + currentState + ); + return gatewayMetaState; + } + private static BigArrays getBigArrays() { return usually() ? BigArrays.NON_RECYCLING_INSTANCE From dc4dbced2f9046ab5debb5dfec4e9634217be0c2 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 25 Sep 2024 15:27:25 +0530 Subject: [PATCH 054/108] Optimize remote store GC flow with pinned timestamps (#15943) Signed-off-by: Sachin Kale --- ...rePinnedTimestampsGarbageCollectionIT.java | 433 ++++++++++++++++++ .../snapshots/DeleteSnapshotV2IT.java | 231 +++++----- .../store/RemoteSegmentStoreDirectory.java | 16 +- .../RemoteFsTimestampAwareTranslog.java | 171 ++++--- .../index/translog/RemoteFsTranslog.java | 13 +- .../transfer/TranslogTransferMetadata.java | 10 + .../RemoteStorePinnedTimestampService.java | 4 + .../blobstore/BlobStoreRepository.java | 43 +- .../snapshots/SnapshotsService.java | 10 +- .../RemoteSegmentStoreDirectoryTests.java | 3 +- .../RemoteFsTimestampAwareTranslogTests.java | 147 +++++- 11 files changed, 854 insertions(+), 227 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java new file mode 100644 index 0000000000000..0a2668c60d3bd --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java @@ -0,0 +1,433 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore; + +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.translog.transfer.TranslogTransferMetadata; +import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING; +import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; +import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; +import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RemoteStorePinnedTimestampsGarbageCollectionIT extends RemoteStoreBaseIntegTestCase { + static final String INDEX_NAME = "remote-store-test-idx-1"; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), false) + .build(); + } + + private void keepPinnedTimestampSchedulerUpdated() throws InterruptedException { + long currentTime = System.currentTimeMillis(); + int maxRetry = 10; + while (maxRetry > 0 && RemoteStorePinnedTimestampService.getPinnedTimestamps().v1() <= currentTime) { + Thread.sleep(1000); + maxRetry--; + } + } + + ActionListener noOpActionListener = new ActionListener<>() { + @Override + public void onResponse(Void unused) {} + + @Override + public void onFailure(Exception e) {} + }; + + public void testLiveIndexNoPinnedTimestamps() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(2, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }); + } + + public void testLiveIndexNoPinnedTimestampsWithExtraGenSettingWithinLimit() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 10) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 9); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertTrue(metadataFiles.size() >= numDocs + 1); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }, 30, TimeUnit.SECONDS); + } + + public void testLiveIndexNoPinnedTimestampsWithExtraGenSetting() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 3) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = 5; + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(5, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }); + } + + public void testLiveIndexWithPinnedTimestamps() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + if (i == 2) { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueMinutes(1)); + remoteStorePinnedTimestampService.pinTimestamp(System.currentTimeMillis(), "xyz", noOpActionListener); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + } + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(3, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }); + } + + public void testIndexDeletionNoPinnedTimestamps() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(2, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }); + + keepPinnedTimestampSchedulerUpdated(); + client().admin().indices().prepareDelete(INDEX_NAME).get(); + + assertBusy(() -> { + assertEquals(0, Files.list(translogMetadataPath).collect(Collectors.toList()).size()); + assertEquals(0, Files.list(translogDataPath).collect(Collectors.toList()).size()); + }); + } + + public void testIndexDeletionWithPinnedTimestamps() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + if (i == 2) { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueMinutes(1)); + remoteStorePinnedTimestampService.pinTimestamp(System.currentTimeMillis(), "xyz", noOpActionListener); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + } + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(3, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }, 30, TimeUnit.SECONDS); + + keepPinnedTimestampSchedulerUpdated(); + client().admin().indices().prepareDelete(INDEX_NAME).get(); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(1, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }); + } + + private void verifyTranslogDataFileCount(List metadataFiles, Path translogDataPath) throws IOException { + List mdFiles = metadataFiles.stream().map(p -> p.getFileName().toString()).collect(Collectors.toList()); + Set generations = new HashSet<>(); + for (String mdFile : mdFiles) { + Tuple minMaxGen = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(mdFile); + generations.addAll(LongStream.rangeClosed(minMaxGen.v1(), minMaxGen.v2()).boxed().collect(Collectors.toList())); + } + assertEquals(generations.size() * 2, Files.list(translogDataPath).collect(Collectors.toList()).size()); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java index 1d7a58384c0be..c4e3a478c8540 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java @@ -23,19 +23,31 @@ import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.lessThan; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class DeleteSnapshotV2IT extends AbstractSnapshotIntegTestCase { private static final String REMOTE_REPO_NAME = "remote-store-repo-name"; + private void keepPinnedTimestampSchedulerUpdated() throws InterruptedException { + long currentTime = System.currentTimeMillis(); + int maxRetry = 10; + while (maxRetry > 0 && RemoteStorePinnedTimestampService.getPinnedTimestamps().v1() <= currentTime) { + Thread.sleep(1000); + maxRetry--; + } + } + public void testDeleteShallowCopyV2() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); @@ -74,8 +86,8 @@ public void testDeleteShallowCopyV2() throws Exception { createIndex(indexName1, getRemoteStoreBackedIndexSettings()); createIndex(indexName2, getRemoteStoreBackedIndexSettings()); - final int numDocsInIndex1 = 10; - final int numDocsInIndex2 = 20; + final int numDocsInIndex1 = 1; + final int numDocsInIndex2 = 2; indexRandomDocs(indexName1, numDocsInIndex1); indexRandomDocs(indexName2, numDocsInIndex2); ensureGreen(indexName1, indexName2); @@ -92,7 +104,7 @@ public void testDeleteShallowCopyV2() throws Exception { assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName1)); createIndex(indexName3, getRemoteStoreBackedIndexSettings()); - indexRandomDocs(indexName3, 10); + indexRandomDocs(indexName3, 1); CreateSnapshotResponse createSnapshotResponse2 = client().admin() .cluster() .prepareCreateSnapshot(snapshotRepoName, snapshotName2) @@ -105,109 +117,101 @@ public void testDeleteShallowCopyV2() throws Exception { assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2)); assertAcked(client().admin().indices().prepareDelete(indexName1)); - Thread.sleep(100); - AcknowledgedResponse deleteResponse = client().admin() - .cluster() - .prepareDeleteSnapshot(snapshotRepoName, snapshotName2) - .setSnapshots(snapshotName2) - .get(); + AcknowledgedResponse deleteResponse = client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, snapshotName2).get(); assertTrue(deleteResponse.isAcknowledged()); // test delete non-existent snapshot assertThrows( SnapshotMissingException.class, - () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").setSnapshots(snapshotName2).get() + () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").get() ); - } - public void testDeleteShallowCopyV2MultipleSnapshots() throws Exception { + public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); + Settings settings = remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath); + settings = Settings.builder() + .put(settings) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.FIXED.toString()) + .build(); + String clusterManagerName = internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNode(settings); + final Client clusterManagerClient = internalCluster().clusterManagerClient(); + ensureStableCluster(2); - internalCluster().startClusterManagerOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); - internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); - internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); - - String indexName1 = "testindex1"; - String indexName2 = "testindex2"; - String indexName3 = "testindex3"; - String snapshotRepoName = "test-create-snapshot-repo"; - String snapshotName1 = "test-create-snapshot1"; - String snapshotName2 = "test-create-snapshot2"; - Path absolutePath1 = randomRepoPath().toAbsolutePath(); - logger.info("Snapshot Path [{}]", absolutePath1); - - Client client = client(); - - assertAcked( - client.admin() - .cluster() - .preparePutRepository(snapshotRepoName) - .setType(FsRepository.TYPE) - .setSettings( - Settings.builder() - .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) - .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) - .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) - .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) - .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true) - ) + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + clusterManagerName ); - createIndex(indexName1, getRemoteStoreBackedIndexSettings()); + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowV2(snapshotRepoPath)); - createIndex(indexName2, getRemoteStoreBackedIndexSettings()); + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, 25); - final int numDocsInIndex1 = 10; - final int numDocsInIndex2 = 20; - indexRandomDocs(indexName1, numDocsInIndex1); - indexRandomDocs(indexName2, numDocsInIndex2); - ensureGreen(indexName1, indexName2); + String indexUUID = client().admin() + .indices() + .prepareGetSettings(remoteStoreEnabledIndexName) + .get() + .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_INDEX_UUID); + logger.info("--> create two remote index shallow snapshots"); CreateSnapshotResponse createSnapshotResponse = client().admin() .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .prepareCreateSnapshot(snapshotRepoName, "snap1") .setWaitForCompletion(true) .get(); - SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); - assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); - assertThat(snapshotInfo.successfulShards(), greaterThan(0)); - assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); - assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName1)); + SnapshotInfo snapshotInfo1 = createSnapshotResponse.getSnapshotInfo(); - createIndex(indexName3, getRemoteStoreBackedIndexSettings()); - indexRandomDocs(indexName3, 10); + Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID); + Path shardPath = Path.of(String.valueOf(indexPath), "0"); - CreateSnapshotResponse createSnapshotResponse2 = client().admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName2) - .setWaitForCompletion(true) - .get(); - snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); - assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); - assertThat(snapshotInfo.successfulShards(), greaterThan(0)); - assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); - assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2)); + // delete remote store index + assertAcked(client().admin().indices().prepareDelete(remoteStoreEnabledIndexName)); + + logger.info("--> delete snapshot 1"); + + Path segmentsPath = Path.of(String.valueOf(shardPath), "segments"); + Path translogPath = Path.of(String.valueOf(shardPath), "translog"); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); - AcknowledgedResponse deleteResponse = client().admin() + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + keepPinnedTimestampSchedulerUpdated(); + + AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin() .cluster() - .prepareDeleteSnapshot(snapshotRepoName, snapshotName1, snapshotName2) - .setSnapshots(snapshotName2) + .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo1.snapshotId().getName()) .get(); - assertTrue(deleteResponse.isAcknowledged()); + assertAcked(deleteSnapshotResponse); - // test delete non-existent snapshot - assertThrows( - SnapshotMissingException.class, - () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").setSnapshots(snapshotName2).get() - ); + // Delete is async. Give time for it + assertBusy(() -> { + try { + assertEquals(0, RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath)); + } catch (NoSuchFileException e) { + fail(); + } + }, 60, TimeUnit.SECONDS); + assertBusy(() -> { + try { + assertEquals(0, RemoteStoreBaseIntegTestCase.getFileCount(translogPath)); + } catch (NoSuchFileException e) { + fail(); + } + }, 60, TimeUnit.SECONDS); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15692") - public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exception { + public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2MultipleSnapshots() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); Settings settings = remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath); @@ -242,11 +246,11 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio .get() .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_INDEX_UUID); - String numShards = client().admin() - .indices() - .prepareGetSettings(remoteStoreEnabledIndexName) - .get() - .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_NUMBER_OF_SHARDS); + Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID); + Path shardPath = Path.of(String.valueOf(indexPath), "0"); + + Path segmentsPath = Path.of(String.valueOf(shardPath), "segments", "data"); + Path translogPath = Path.of(String.valueOf(shardPath), "translog", "data", "1"); logger.info("--> create two remote index shallow snapshots"); CreateSnapshotResponse createSnapshotResponse = client().admin() @@ -256,6 +260,11 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio .get(); SnapshotInfo snapshotInfo1 = createSnapshotResponse.getSnapshotInfo(); + List segmentsPostSnapshot1 = Files.list(segmentsPath).collect(Collectors.toList()); + List translogPostSnapshot1 = Files.list(translogPath).collect(Collectors.toList()); + + forceMerge(1); + refresh(remoteStoreEnabledIndexName); indexRandomDocs(remoteStoreEnabledIndexName, 25); CreateSnapshotResponse createSnapshotResponse2 = client().admin() @@ -264,46 +273,28 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio .setWaitForCompletion(true) .get(); SnapshotInfo snapshotInfo2 = createSnapshotResponse2.getSnapshotInfo(); + + List segmentsPostSnapshot2 = Files.list(segmentsPath).collect(Collectors.toList()); + List translogPostSnapshot2 = Files.list(translogPath).collect(Collectors.toList()); + assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfo2.successfulShards(), greaterThan(0)); assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards())); assertThat(snapshotInfo2.snapshotId().getName(), equalTo("snap2")); - // delete remote store index - assertAcked(client().admin().indices().prepareDelete(remoteStoreEnabledIndexName)); - - logger.info("--> delete snapshot 2"); + assertBusy(() -> assertTrue(translogPostSnapshot2.size() > translogPostSnapshot1.size()), 60, TimeUnit.SECONDS); + assertBusy(() -> assertTrue(segmentsPostSnapshot2.size() > segmentsPostSnapshot1.size()), 60, TimeUnit.SECONDS); - Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID); - Path shardPath = Path.of(String.valueOf(indexPath), "0"); - Path segmentsPath = Path.of(String.valueOf(shardPath), "segments"); - Path translogPath = Path.of(String.valueOf(shardPath), "translog"); - - // Get total segments remote store directory file count for deleted index and shard 0 - int segmentFilesCountBeforeDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath); - int translogFilesCountBeforeDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(translogPath); - - RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + keepPinnedTimestampSchedulerUpdated(); - AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin() - .cluster() - .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo2.snapshotId().getName()) - .get(); - assertAcked(deleteSnapshotResponse); - - Thread.sleep(5000); - - assertBusy(() -> { - try { - assertThat(RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath), lessThan(segmentFilesCountBeforeDeletingSnapshot1)); - } catch (Exception e) {} - }, 30, TimeUnit.SECONDS); - int segmentFilesCountAfterDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath); + // delete remote store index + assertAcked(client().admin().indices().prepareDelete(remoteStoreEnabledIndexName)); logger.info("--> delete snapshot 1"); RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + keepPinnedTimestampSchedulerUpdated(); // on snapshot deletion, remote store segment files should get cleaned up for deleted index - `remote-index-1` - deleteSnapshotResponse = clusterManagerClient.admin() + AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin() .cluster() .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo1.snapshotId().getName()) .get(); @@ -311,23 +302,21 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio // Delete is async. Give time for it assertBusy(() -> { - try { - assertThat(RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath), lessThan(segmentFilesCountAfterDeletingSnapshot1)); - } catch (Exception e) {} + List segmentsPostDeletionOfSnapshot1 = Files.list(segmentsPath).collect(Collectors.toList()); + assertTrue(segmentsPostDeletionOfSnapshot1.size() < segmentsPostSnapshot2.size()); }, 60, TimeUnit.SECONDS); - - assertBusy(() -> { - try { - assertThat(RemoteStoreBaseIntegTestCase.getFileCount(translogPath), lessThan(translogFilesCountBeforeDeletingSnapshot1)); - } catch (Exception e) {} - }, 60, TimeUnit.SECONDS); - + // To uncomment following, we need to handle deletion of generations in translog cleanup flow + // List translogPostDeletionOfSnapshot1 = Files.list(translogPath).collect(Collectors.toList()); + // Delete is async. Give time for it + // assertBusy(() -> assertEquals(translogPostSnapshot2.size() - translogPostSnapshot1.size(), + // translogPostDeletionOfSnapshot1.size()), 60, TimeUnit.SECONDS); } private Settings snapshotV2Settings(Path remoteStoreRepoPath) { Settings settings = Settings.builder() .put(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)) .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), false) .build(); return settings; } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 72bf07d4b03b2..5be516166803e 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -854,7 +854,7 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException } // Check last fetch status of pinned timestamps. If stale, return. - if (RemoteStoreUtils.isPinnedTimestampStateStale()) { + if (lastNMetadataFilesToKeep != 0 && RemoteStoreUtils.isPinnedTimestampStateStale()) { logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); return; } @@ -994,7 +994,8 @@ public static void remoteDirectoryCleanup( String remoteStoreRepoForIndex, String indexUUID, ShardId shardId, - RemoteStorePathStrategy pathStrategy + RemoteStorePathStrategy pathStrategy, + boolean forceClean ) { try { RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = (RemoteSegmentStoreDirectory) remoteDirectoryFactory.newDirectory( @@ -1003,8 +1004,12 @@ public static void remoteDirectoryCleanup( shardId, pathStrategy ); - remoteSegmentStoreDirectory.deleteStaleSegments(0); - remoteSegmentStoreDirectory.deleteIfEmpty(); + if (forceClean) { + remoteSegmentStoreDirectory.delete(); + } else { + remoteSegmentStoreDirectory.deleteStaleSegments(0); + remoteSegmentStoreDirectory.deleteIfEmpty(); + } } catch (Exception e) { staticLogger.error("Exception occurred while deleting directory", e); } @@ -1023,7 +1028,10 @@ private boolean deleteIfEmpty() throws IOException { logger.info("Remote directory still has files, not deleting the path"); return false; } + return delete(); + } + private boolean delete() { try { remoteDataDirectory.delete(); remoteMetadataDirectory.delete(); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index 27d34ec0d05af..e61a9606175ee 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -34,6 +34,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BooleanSupplier; import java.util.function.LongConsumer; @@ -61,6 +62,7 @@ public class RemoteFsTimestampAwareTranslog extends RemoteFsTranslog { private final Map> oldFormatMetadataFileGenerationMap; private final Map> oldFormatMetadataFilePrimaryTermMap; private final AtomicLong minPrimaryTermInRemote = new AtomicLong(Long.MAX_VALUE); + private final AtomicBoolean triggerTrimOnMinRemoteGenReferencedChange = new AtomicBoolean(false); public RemoteFsTimestampAwareTranslog( TranslogConfig config, @@ -105,6 +107,11 @@ protected void onDelete() { } } + @Override + protected void onMinRemoteGenReferencedChange() { + triggerTrimOnMinRemoteGenReferencedChange.set(true); + } + @Override public void trimUnreferencedReaders() throws IOException { trimUnreferencedReaders(false, true); @@ -135,14 +142,22 @@ protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal) // This is to ensure that after the permits are acquired during primary relocation, there are no further modification on remote // store. - if (startedPrimarySupplier.getAsBoolean() == false || pauseSync.get()) { + if (indexDeleted == false && (startedPrimarySupplier.getAsBoolean() == false || pauseSync.get())) { return; } // This is to fail fast and avoid listing md files un-necessarily. if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) { - logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); + logger.warn("Skipping remote translog garbage collection as last fetch of pinned timestamp is stale"); + return; + } + + // This code block ensures parity with RemoteFsTranslog. Without this, we will end up making list translog metadata + // call in each invocation of trimUnreferencedReaders + if (indexDeleted == false && triggerTrimOnMinRemoteGenReferencedChange.get() == false) { return; + } else if (triggerTrimOnMinRemoteGenReferencedChange.get()) { + triggerTrimOnMinRemoteGenReferencedChange.set(false); } // Since remote generation deletion is async, this ensures that only one generation deletion happens at a time. @@ -158,7 +173,7 @@ public void onResponse(List blobMetadata) { List metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); try { - if (metadataFiles.size() <= 1) { + if (indexDeleted == false && metadataFiles.size() <= 1) { logger.debug("No stale translog metadata files found"); remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); return; @@ -166,16 +181,12 @@ public void onResponse(List blobMetadata) { // Check last fetch status of pinned timestamps. If stale, return. if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) { - logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); + logger.warn("Skipping remote translog garbage collection as last fetch of pinned timestamp is stale"); remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); return; } - List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted( - metadataFiles, - metadataFilePinnedTimestampMap, - logger - ); + List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, indexDeleted); // If index is not deleted, make sure to keep latest metadata file if (indexDeleted == false) { @@ -194,10 +205,11 @@ public void onResponse(List blobMetadata) { metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); logger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted); + Set generationsToBeDeleted = getGenerationsToBeDeleted( metadataFilesNotToBeDeleted, metadataFilesToBeDeleted, - indexDeleted + indexDeleted ? Long.MAX_VALUE : getMinGenerationToKeepInRemote() ); logger.debug(() -> "generationsToBeDeleted = " + generationsToBeDeleted); @@ -208,7 +220,11 @@ public void onResponse(List blobMetadata) { generationsToBeDeleted, remoteGenerationDeletionPermits::release ); + } else { + remoteGenerationDeletionPermits.release(); + } + if (metadataFilesToBeDeleted.isEmpty() == false) { // Delete stale metadata files translogTransferManager.deleteMetadataFilesAsync( metadataFilesToBeDeleted, @@ -217,11 +233,10 @@ public void onResponse(List blobMetadata) { // Update cache to keep only those metadata files that are not getting deleted oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted); - // Delete stale primary terms deleteStaleRemotePrimaryTerms(metadataFilesNotToBeDeleted); } else { - remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + remoteGenerationDeletionPermits.release(); } } catch (Exception e) { remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); @@ -237,18 +252,16 @@ public void onFailure(Exception e) { translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener); } + private long getMinGenerationToKeepInRemote() { + return minRemoteGenReferenced - indexSettings().getRemoteTranslogExtraKeep(); + } + // Visible for testing protected Set getGenerationsToBeDeleted( List metadataFilesNotToBeDeleted, List metadataFilesToBeDeleted, - boolean indexDeleted + long minGenerationToKeepInRemote ) throws IOException { - long maxGenerationToBeDeleted = Long.MAX_VALUE; - - if (indexDeleted == false) { - maxGenerationToBeDeleted = minRemoteGenReferenced - 1 - indexSettings().getRemoteTranslogExtraKeep(); - } - Set generationsFromMetadataFilesToBeDeleted = new HashSet<>(); for (String mdFile : metadataFilesToBeDeleted) { Tuple minMaxGen = getMinMaxTranslogGenerationFromMetadataFile(mdFile, translogTransferManager); @@ -262,21 +275,31 @@ protected Set getGenerationsToBeDeleted( Set generationsToBeDeleted = new HashSet<>(); for (long generation : generationsFromMetadataFilesToBeDeleted) { // Check if the generation is not referred by metadata file matching pinned timestamps - if (generation <= maxGenerationToBeDeleted && isGenerationPinned(generation, pinnedGenerations) == false) { + // The check with minGenerationToKeep is redundant but kept as to make sure we don't delete generations + // that are not persisted in remote segment store yet. + if (generation < minGenerationToKeepInRemote && isGenerationPinned(generation, pinnedGenerations) == false) { generationsToBeDeleted.add(generation); } } return generationsToBeDeleted; } - protected List getMetadataFilesToBeDeleted(List metadataFiles) { - return getMetadataFilesToBeDeleted(metadataFiles, metadataFilePinnedTimestampMap, logger); + protected List getMetadataFilesToBeDeleted(List metadataFiles, boolean indexDeleted) { + return getMetadataFilesToBeDeleted( + metadataFiles, + metadataFilePinnedTimestampMap, + getMinGenerationToKeepInRemote(), + indexDeleted, + logger + ); } // Visible for testing protected static List getMetadataFilesToBeDeleted( List metadataFiles, Map metadataFilePinnedTimestampMap, + long minGenerationToKeepInRemote, + boolean indexDeleted, Logger logger ) { Tuple> pinnedTimestampsState = RemoteStorePinnedTimestampService.getPinnedTimestamps(); @@ -312,6 +335,22 @@ protected static List getMetadataFilesToBeDeleted( metadataFilesToBeDeleted.size() ); + if (indexDeleted == false) { + // Filter out metadata files based on minGenerationToKeep + List metadataFilesContainingMinGenerationToKeep = metadataFilesToBeDeleted.stream().filter(md -> { + long maxGeneration = TranslogTransferMetadata.getMaxGenerationFromFileName(md); + return maxGeneration == -1 || maxGeneration >= minGenerationToKeepInRemote; + }).collect(Collectors.toList()); + metadataFilesToBeDeleted.removeAll(metadataFilesContainingMinGenerationToKeep); + + logger.trace( + "metadataFilesContainingMinGenerationToKeep.size = {}, metadataFilesToBeDeleted based on minGenerationToKeep filtering = {}, minGenerationToKeep = {}", + metadataFilesContainingMinGenerationToKeep.size(), + metadataFilesToBeDeleted.size(), + minGenerationToKeepInRemote + ); + } + return metadataFilesToBeDeleted; } @@ -472,50 +511,60 @@ protected static Tuple getMinMaxPrimaryTermFromMetadataFile( } } - public static void cleanup(TranslogTransferManager translogTransferManager) throws IOException { - ActionListener> listMetadataFilesListener = new ActionListener<>() { - @Override - public void onResponse(List blobMetadata) { - List metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); + public static void cleanupOfDeletedIndex(TranslogTransferManager translogTransferManager, boolean forceClean) throws IOException { + if (forceClean) { + translogTransferManager.delete(); + } else { + ActionListener> listMetadataFilesListener = new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + List metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); + + try { + if (metadataFiles.isEmpty()) { + staticLogger.debug("No stale translog metadata files found"); + return; + } + List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + Long.MAX_VALUE, + true, // This method gets called when the index is no longer present + staticLogger + ); + if (metadataFilesToBeDeleted.isEmpty()) { + staticLogger.debug("No metadata files to delete"); + return; + } + staticLogger.debug(() -> "metadataFilesToBeDeleted = " + metadataFilesToBeDeleted); - try { - if (metadataFiles.isEmpty()) { - staticLogger.debug("No stale translog metadata files found"); - return; - } - List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, new HashMap<>(), staticLogger); - if (metadataFilesToBeDeleted.isEmpty()) { - staticLogger.debug("No metadata files to delete"); - return; - } - staticLogger.debug(() -> "metadataFilesToBeDeleted = " + metadataFilesToBeDeleted); + // For all the files that we are keeping, fetch min and max generations + List metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles); + metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); + staticLogger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted); - // For all the files that we are keeping, fetch min and max generations - List metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles); - metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); - staticLogger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted); + // Delete stale metadata files + translogTransferManager.deleteMetadataFilesAsync(metadataFilesToBeDeleted, () -> {}); - // Delete stale metadata files - translogTransferManager.deleteMetadataFilesAsync(metadataFilesToBeDeleted, () -> {}); + // Delete stale primary terms + deleteStaleRemotePrimaryTerms( + metadataFilesNotToBeDeleted, + translogTransferManager, + new HashMap<>(), + new AtomicLong(Long.MAX_VALUE), + staticLogger + ); + } catch (Exception e) { + staticLogger.error("Exception while cleaning up metadata and primary terms", e); + } + } - // Delete stale primary terms - deleteStaleRemotePrimaryTerms( - metadataFilesNotToBeDeleted, - translogTransferManager, - new HashMap<>(), - new AtomicLong(Long.MAX_VALUE), - staticLogger - ); - } catch (Exception e) { + @Override + public void onFailure(Exception e) { staticLogger.error("Exception while cleaning up metadata and primary terms", e); } - } - - @Override - public void onFailure(Exception e) { - staticLogger.error("Exception while cleaning up metadata and primary terms", e); - } - }; - translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener); + }; + translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener); + } } } diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 812852d107682..85f58f898826f 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -683,12 +683,17 @@ private class RemoteFsTranslogTransferListener implements TranslogTransferListen @Override public void onUploadComplete(TransferSnapshot transferSnapshot) throws IOException { maxRemoteTranslogGenerationUploaded = generation; + long previousMinRemoteGenReferenced = minRemoteGenReferenced; minRemoteGenReferenced = getMinFileGeneration(); + if (previousMinRemoteGenReferenced != minRemoteGenReferenced) { + onMinRemoteGenReferencedChange(); + } logger.debug( - "Successfully uploaded translog for primary term = {}, generation = {}, maxSeqNo = {}", + "Successfully uploaded translog for primary term = {}, generation = {}, maxSeqNo = {}, minRemoteGenReferenced = {}", primaryTerm, generation, - maxSeqNo + maxSeqNo, + minRemoteGenReferenced ); } @@ -702,6 +707,10 @@ public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) thro } } + protected void onMinRemoteGenReferencedChange() { + + } + @Override public long getMinUnreferencedSeqNoInSegments(long minUnrefCheckpointInLastCommit) { return minSeqNoToKeep; diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java index 3b8885055e8f7..7fe3305545085 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java @@ -170,6 +170,16 @@ public static Tuple getMinMaxTranslogGenerationFromFilename(String f } } + public static long getMaxGenerationFromFileName(String filename) { + String[] tokens = filename.split(METADATA_SEPARATOR); + try { + return RemoteStoreUtils.invertLong(tokens[2]); + } catch (Exception e) { + logger.error(() -> new ParameterizedMessage("Exception while getting max generation from: {}", filename), e); + return -1; + } + } + public static Tuple getMinMaxPrimaryTermFromFilename(String filename) { String[] tokens = filename.split(METADATA_SEPARATOR); if (tokens.length < 7) { diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index 71133615ed056..1448c46583f6a 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -240,6 +240,10 @@ public void unpinTimestamp(long timestamp, String pinningEntity, ActionListener< } } + public void forceSyncPinnedTimestamps() { + asyncUpdatePinnedTimestampTask.run(); + } + @Override public void close() throws IOException { asyncUpdatePinnedTimestampTask.close(); diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 0292cecc36a81..14c201e819994 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -199,7 +199,6 @@ import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1; import static org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName; import static org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS; -import static org.opensearch.snapshots.SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER; /** * BlobStore - based implementation of Snapshot Repository @@ -1309,7 +1308,7 @@ private void cleanUpRemoteStoreFilesForDeletedIndicesV2( } // iterate through all the indices and trigger remote store directory cleanup for deleted index segments for (String indexId : uniqueIndexIds) { - cleanRemoteStoreDirectoryIfNeeded(snapshotIds, indexId, repositoryData, remoteSegmentStoreDirectoryFactory); + cleanRemoteStoreDirectoryIfNeeded(snapshotIds, indexId, repositoryData, remoteSegmentStoreDirectoryFactory, false); } afterCleanupsListener.onResponse(null); } catch (Exception e) { @@ -1357,11 +1356,17 @@ private void removeSnapshotPinnedTimestamp( ) { remoteStorePinnedTimestampService.unpinTimestamp( timestampToUnpin, - repository + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshotId.getUUID(), + SnapshotsService.getPinningEntity(repository, snapshotId.getUUID()), new ActionListener() { @Override public void onResponse(Void unused) { - logger.debug("Timestamp {} unpinned successfully for snapshot {}", timestampToUnpin, snapshotId.getName()); + logger.info("Timestamp {} unpinned successfully for snapshot {}", timestampToUnpin, snapshotId.getName()); + try { + remoteStorePinnedTimestampService.forceSyncPinnedTimestamps(); + logger.debug("Successfully synced pinned timestamp state"); + } catch (Exception e) { + logger.warn("Exception while updating pinning timestamp state, snapshot deletion will continue", e); + } listener.onResponse(null); } @@ -1466,7 +1471,8 @@ public static void remoteDirectoryCleanupAsync( String indexUUID, ShardId shardId, String threadPoolName, - RemoteStorePathStrategy pathStrategy + RemoteStorePathStrategy pathStrategy, + boolean forceClean ) { threadpool.executor(threadPoolName) .execute( @@ -1476,7 +1482,8 @@ public static void remoteDirectoryCleanupAsync( remoteStoreRepoForIndex, indexUUID, shardId, - pathStrategy + pathStrategy, + forceClean ), indexUUID, shardId @@ -1532,7 +1539,8 @@ protected void releaseRemoteStoreLockAndCleanup( indexUUID, new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.parseInt(shardId)), ThreadPool.Names.REMOTE_PURGE, - remoteStoreShardShallowCopySnapshot.getRemoteStorePathStrategy() + remoteStoreShardShallowCopySnapshot.getRemoteStorePathStrategy(), + false ); } } @@ -2095,7 +2103,7 @@ private void executeOneStaleIndexDelete( deleteResult = deleteResult.add(cleanUpStaleSnapshotShardPathsFile(matchingShardPaths, snapshotShardPaths)); if (remoteSegmentStoreDirectoryFactory != null) { - cleanRemoteStoreDirectoryIfNeeded(deletedSnapshots, indexSnId, oldRepoData, remoteSegmentStoreDirectoryFactory); + cleanRemoteStoreDirectoryIfNeeded(deletedSnapshots, indexSnId, oldRepoData, remoteSegmentStoreDirectoryFactory, true); } // Finally, we delete the [base_path]/indexId folder @@ -2167,7 +2175,8 @@ private void cleanRemoteStoreDirectoryIfNeeded( Collection deletedSnapshots, String indexSnId, RepositoryData oldRepoData, - RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, + boolean forceClean ) { assert (indexSnId != null); @@ -2219,9 +2228,16 @@ private void cleanRemoteStoreDirectoryIfNeeded( prevIndexMetadata.getIndexUUID(), shard, ThreadPool.Names.REMOTE_PURGE, - remoteStorePathStrategy + remoteStorePathStrategy, + forceClean + ); + remoteTranslogCleanupAsync( + remoteTranslogRepository, + shard, + remoteStorePathStrategy, + prevIndexMetadata, + forceClean ); - remoteTranslogCleanupAsync(remoteTranslogRepository, shard, remoteStorePathStrategy, prevIndexMetadata); } } } catch (Exception e) { @@ -2245,7 +2261,8 @@ private void remoteTranslogCleanupAsync( Repository remoteTranslogRepository, ShardId shardId, RemoteStorePathStrategy remoteStorePathStrategy, - IndexMetadata prevIndexMetadata + IndexMetadata prevIndexMetadata, + boolean forceClean ) { assert remoteTranslogRepository instanceof BlobStoreRepository; boolean indexMetadataEnabled = RemoteStoreUtils.determineTranslogMetadataEnabled(prevIndexMetadata); @@ -2262,7 +2279,7 @@ private void remoteTranslogCleanupAsync( indexMetadataEnabled ); try { - RemoteFsTimestampAwareTranslog.cleanup(translogTransferManager); + RemoteFsTimestampAwareTranslog.cleanupOfDeletedIndex(translogTransferManager, forceClean); } catch (IOException e) { logger.error("Exception while cleaning up remote translog for shard: " + shardId, e); } diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index f6e550525a3e5..23f6deff3715d 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -613,7 +613,7 @@ private void updateSnapshotPinnedTimestamp( ) { remoteStorePinnedTimestampService.pinTimestamp( timestampToPin, - snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshot.getSnapshotId().getUUID(), + getPinningEntity(snapshot.getRepository(), snapshot.getSnapshotId().getUUID()), new ActionListener() { @Override public void onResponse(Void unused) { @@ -631,6 +631,10 @@ public void onFailure(Exception e) { ); } + public static String getPinningEntity(String repositoryName, String snapshotUUID) { + return repositoryName + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshotUUID; + } + private void cloneSnapshotPinnedTimestamp( RepositoryData repositoryData, SnapshotId sourceSnapshot, @@ -640,8 +644,8 @@ private void cloneSnapshotPinnedTimestamp( ) { remoteStorePinnedTimestampService.cloneTimestamp( timestampToPin, - snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + sourceSnapshot.getUUID(), - snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshot.getSnapshotId().getUUID(), + getPinningEntity(snapshot.getRepository(), sourceSnapshot.getUUID()), + getPinningEntity(snapshot.getRepository(), snapshot.getSnapshotId().getUUID()), new ActionListener() { @Override public void onResponse(Void unused) { diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index ecd6620dbea15..df3df81361a12 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -565,7 +565,8 @@ public void testCleanupAsync() throws Exception { repositoryName, indexUUID, shardId, - pathStrategy + pathStrategy, + false ); verify(remoteSegmentStoreDirectoryFactory).newDirectory(repositoryName, indexUUID, shardId, pathStrategy); verify(threadPool, times(0)).executor(ThreadPool.Names.REMOTE_PURGE); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 4ec68f7fb75b4..e6871414cf5e0 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -214,6 +214,7 @@ public void onFailure(Exception e) { // Old format metadata file String oldFormatMdFilename = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1"; assertNull(TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(oldFormatMdFilename)); + assertEquals(Long.MAX_VALUE - 9223372036854774799L, TranslogTransferMetadata.getMaxGenerationFromFileName(oldFormatMdFilename)); // Node id containing separator String nodeIdWithSeparator = @@ -221,10 +222,14 @@ public void onFailure(Exception e) { Tuple minMaxGen = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(nodeIdWithSeparator); Long minGen = Long.MAX_VALUE - 9223372036438563958L; assertEquals(minGen, minMaxGen.v1()); + Long maxGen = Long.MAX_VALUE - 9223372036854774799L; + assertEquals(maxGen, minMaxGen.v2()); + assertEquals(Long.MAX_VALUE - 9223372036854774799L, TranslogTransferMetadata.getMaxGenerationFromFileName(nodeIdWithSeparator)); // Malformed md filename String malformedMdFileName = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__node1__xyz__3__1"; assertNull(TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(malformedMdFileName)); + assertEquals(Long.MAX_VALUE - 9223372036854774799L, TranslogTransferMetadata.getMaxGenerationFromFileName(malformedMdFileName)); } public void testGetMinMaxPrimaryTermFromFilename() throws Exception { @@ -330,43 +335,60 @@ public void testSimpleOperationsUpload() throws Exception { addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 })); addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 3, primaryTerm.get(), new byte[] { 1 })); - addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); - addToTranslogAndListAndUpload(translog, ops, new Translog.Index("5", 5, primaryTerm.get(), new byte[] { 1 })); - addToTranslogAndListAndUpload(translog, ops, new Translog.Index("6", 6, primaryTerm.get(), new byte[] { 1 })); assertBusy(() -> { assertEquals( - 16, + 10, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() ); }); - assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); - RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); // Fetch pinned timestamps so that it won't be stale updatePinnedTimstampTask.run(); + translog.setMinSeqNoToKeep(3); + translog.trimUnreferencedReaders(); - translog.setMinSeqNoToKeep(4); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("5", 5, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("6", 6, primaryTerm.get(), new byte[] { 1 })); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.setMinSeqNoToKeep(6); translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + + assertEquals(1, translog.readers.size()); + assertBusy(() -> { + assertEquals(2, translog.allUploaded().size()); + assertEquals(4, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals( + 16, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }, 30, TimeUnit.SECONDS); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("7", 7, primaryTerm.get(), new byte[] { 1 })); addToTranslogAndListAndUpload(translog, ops, new Translog.Index("8", 8, primaryTerm.get(), new byte[] { 1 })); - assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); // Fetch pinned timestamps so that it won't be stale updatePinnedTimstampTask.run(); translog.trimUnreferencedReaders(); - assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); - assertEquals(5, translog.readers.size()); + + assertEquals(3, translog.readers.size()); assertBusy(() -> { - assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); - assertEquals(10, translog.allUploaded().size()); + assertEquals(6, translog.allUploaded().size()); + assertEquals(3, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); assertEquals( - 10, + 12, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() ); - }, 60, TimeUnit.SECONDS); + }, 30, TimeUnit.SECONDS); } @Override @@ -397,7 +419,7 @@ public void testMetadataFileDeletion() throws Exception { ); updatePinnedTimstampTask.run(); translog.trimUnreferencedReaders(); - assertBusy(() -> { assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); }); + assertBusy(() -> { assertEquals(3, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); }); } public void testMetadataFileDeletionWithPinnedTimestamps() throws Exception { @@ -568,7 +590,7 @@ public void testDrainSync() throws Exception { assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertEquals(1, translog.readers.size()); assertBusy(() -> assertEquals(2, translog.allUploaded().size())); - assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); + assertBusy(() -> assertEquals(2, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); } @Override @@ -647,7 +669,7 @@ public void testGetGenerationsToBeDeletedEmptyMetadataFilesNotToBeDeleted() thro Set generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted( metadataFilesNotToBeDeleted, metadataFilesToBeDeleted, - true + Long.MAX_VALUE ); Set md1Generations = LongStream.rangeClosed(4, 7).boxed().collect(Collectors.toSet()); Set md2Generations = LongStream.rangeClosed(17, 37).boxed().collect(Collectors.toSet()); @@ -683,7 +705,7 @@ public void testGetGenerationsToBeDeleted() throws IOException { Set generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted( metadataFilesNotToBeDeleted, metadataFilesToBeDeleted, - true + Long.MAX_VALUE ); Set md1Generations = LongStream.rangeClosed(5, 7).boxed().collect(Collectors.toSet()); Set md2Generations = LongStream.rangeClosed(17, 25).boxed().collect(Collectors.toSet()); @@ -708,7 +730,10 @@ public void testGetMetadataFilesToBeDeletedNoExclusion() { "metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1" ); - assertEquals(metadataFiles, ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles)); + assertEquals( + metadataFiles, + RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(metadataFiles, new HashMap<>(), Long.MAX_VALUE, false, logger) + ); } public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { @@ -724,7 +749,13 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" ); - List metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles); + List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + Long.MAX_VALUE, + false, + logger + ); assertEquals(1, metadataFilesToBeDeleted.size()); assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); } @@ -746,7 +777,13 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnPinningOnly() throws "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" ); - List metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles); + List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + Long.MAX_VALUE, + false, + logger + ); assertEquals(2, metadataFilesToBeDeleted.size()); assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1)); @@ -769,11 +806,77 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeAndPinning() throw "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" ); - List metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles); + List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + Long.MAX_VALUE, + false, + logger + ); assertEquals(1, metadataFilesToBeDeleted.size()); assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(0)); } + public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationOnly() throws IOException { + long currentTimeInMillis = System.currentTimeMillis(); + String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000); + String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); + String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); + + when(blobContainer.listBlobs()).thenReturn(Map.of()); + + updatePinnedTimstampTask.run(); + + List metadataFiles = List.of( + // MaxGen 7 + "metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1", + // MaxGen 12 + "metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1", + // MaxGen 10 + "metadata__9223372036438563903__9223372036854775798__" + md3Timestamp + "__31__9223372036854775701__1" + ); + + List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + 10L, + false, + logger + ); + assertEquals(2, metadataFilesToBeDeleted.size()); + assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); + assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1)); + } + + public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationDeleteIndex() throws IOException { + long currentTimeInMillis = System.currentTimeMillis(); + String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000); + String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); + String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); + + when(blobContainer.listBlobs()).thenReturn(Map.of()); + + updatePinnedTimstampTask.run(); + + List metadataFiles = List.of( + // MaxGen 7 + "metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1", + // MaxGen 12 + "metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1", + // MaxGen 17 + "metadata__9223372036438563903__9223372036854775790__" + md3Timestamp + "__31__9223372036854775701__1" + ); + + List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + 10L, + true, + logger + ); + assertEquals(metadataFiles, metadataFilesToBeDeleted); + } + public void testIsGenerationPinned() { TreeSet> pinnedGenerations = new TreeSet<>(new TreeSet<>((o1, o2) -> { if (Objects.equals(o1.v1(), o2.v1()) == false) { From 0617d95c0e3185c4d5dd38fc9d1637da1b11b0ab Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 25 Sep 2024 18:16:48 +0530 Subject: [PATCH 055/108] Modify flaky test (#15854) Signed-off-by: Shivansh Arora --- .../gateway/remote/RemoteStatePublicationIT.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index ffb9352e8ba47..578c922c80a0d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -235,7 +235,6 @@ public void testRemotePublicationDownloadStats() { assertDataNodeDownloadStats(nodesStatsResponseDataNode); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15767") public void testRemotePublicationDisabledByRollingRestart() throws Exception { prepareCluster(3, 2, INDEX_NAME, 1, 2); ensureStableCluster(5); @@ -272,7 +271,6 @@ public void doAfterNodes(int n, Client client) { assertTrue( stats.getFullClusterStateReceivedCount() > 0 || stats.getCompatibleClusterStateDiffReceivedCount() > 0 ); - assertEquals(0, stats.getIncompatibleClusterStateDiffReceivedCount()); } else { DiscoveryStats stats = nodeStats.getDiscoveryStats(); assertEquals(0, stats.getPublishStats().getFullClusterStateReceivedCount()); @@ -297,7 +295,7 @@ public void doAfterNodes(int n, Client client) { ); if (activeCMRestarted) { assertNull(remoteState.getLastAcceptedState()); - // assertNull(remoteState.getLastAcceptedManifest()); + assertNull(remoteState.getLastAcceptedManifest()); } else { ClusterState localState = registry.getPersistedState(PersistedStateRegistry.PersistedStateType.LOCAL) .getLastAcceptedState(); @@ -326,7 +324,6 @@ public void doAfterNodes(int n, Client client) { response.getNodes().forEach(nodeStats -> { PublishClusterStateStats stats = nodeStats.getDiscoveryStats().getPublishStats(); assertTrue(stats.getFullClusterStateReceivedCount() > 0 || stats.getCompatibleClusterStateDiffReceivedCount() > 0); - assertEquals(0, stats.getIncompatibleClusterStateDiffReceivedCount()); }); NodesInfoResponse nodesInfoResponse = client().admin() .cluster() @@ -341,7 +338,7 @@ public void doAfterNodes(int n, Client client) { PersistedStateRegistry registry = internalCluster().getInstance(PersistedStateRegistry.class, node); CoordinationState.PersistedState remoteState = registry.getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE); assertNull(remoteState.getLastAcceptedState()); - // assertNull(remoteState.getLastAcceptedManifest()); + assertNull(remoteState.getLastAcceptedManifest()); }); } From f1acc7aad7db4c3c9ce2e0ac331b02105ddc85f5 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Wed, 25 Sep 2024 18:45:36 +0530 Subject: [PATCH 056/108] Fix issue of red index on close for remote enabled clusters (#15990) * Fix red index on close for remote translog Signed-off-by: Ashish Singh * Add UTs Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- .../opensearch/remotestore/RemoteStoreIT.java | 69 ++++++++++++++++- .../index/translog/RemoteFsTranslog.java | 21 ++++- .../index/translog/RemoteFsTranslogTests.java | 77 +++++++++++++++++++ 3 files changed, 163 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 692727357a88a..ebb911c739eb3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -18,6 +18,7 @@ import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchPhaseExecutionException; +import org.opensearch.client.Requests; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.RecoverySource; @@ -202,7 +203,7 @@ public void testRemoteTranslogCleanup() throws Exception { public void testStaleCommitDeletionWithInvokeFlush() throws Exception { String dataNode = internalCluster().startNode(); - createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000L, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, true, INDEX_NAME); String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); @@ -1011,4 +1012,70 @@ public void testAsyncTranslogDurabilityRestrictionsThroughIdxTemplates() throws .get() ); } + + public void testCloseIndexWithNoOpSyncAndFlushForSyncTranslog() throws InterruptedException { + internalCluster().startNodes(3); + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "5s")) + .get(); + Settings.Builder settings = Settings.builder() + .put(remoteStoreIndexSettings(0, 10000L, -1)) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "1s"); + createIndex(INDEX_NAME, settings.build()); + CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + if (randomBoolean()) { + for (int i = 0; i < randomIntBetween(1, 5); i++) { + indexSingleDoc(INDEX_NAME); + } + flushAndRefresh(INDEX_NAME); + } + // Index single doc to start the asyn io processor to run which will lead to 10s wait time before the next sync. + indexSingleDoc(INDEX_NAME); + // Reduce the latch for the main thread to flush after some sleep. + latch.countDown(); + // Index another doc and in this case the flush would have happened before the sync. + indexSingleDoc(INDEX_NAME); + }).start(); + // Wait for atleast one doc to be ingested. + latch.await(); + // Sleep for some time for the next doc to be present in lucene buffer. If flush happens first before the doc #2 + // gets indexed, then it goes into the happy case where the close index happens succefully. + Thread.sleep(1000); + // Flush so that the subsequent sync or flushes are no-op. + flush(INDEX_NAME); + // Closing the index involves translog.sync and shard.flush which are now no-op. + client().admin().indices().close(Requests.closeIndexRequest(INDEX_NAME)).actionGet(); + Thread.sleep(10000); + ensureGreen(INDEX_NAME); + } + + public void testCloseIndexWithNoOpSyncAndFlushForAsyncTranslog() throws InterruptedException { + internalCluster().startNodes(3); + Settings.Builder settings = Settings.builder() + .put(remoteStoreIndexSettings(0, 10000L, -1)) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "1s") + .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Durability.ASYNC) + .put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "10s"); + createIndex(INDEX_NAME, settings.build()); + CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + // Index some docs to start the asyn io processor to run which will lead to 10s wait time before the next sync. + indexSingleDoc(INDEX_NAME); + indexSingleDoc(INDEX_NAME); + indexSingleDoc(INDEX_NAME); + // Reduce the latch for the main thread to flush after some sleep. + latch.countDown(); + }).start(); + // Wait for atleast one doc to be ingested. + latch.await(); + // Flush so that the subsequent sync or flushes are no-op. + flush(INDEX_NAME); + // Closing the index involves translog.sync and shard.flush which are now no-op. + client().admin().indices().close(Requests.closeIndexRequest(INDEX_NAME)).actionGet(); + Thread.sleep(10000); + ensureGreen(INDEX_NAME); + } } diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 85f58f898826f..80760ed6ddf12 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -78,6 +78,9 @@ public class RemoteFsTranslog extends Translog { // min generation referred by last uploaded translog protected volatile long minRemoteGenReferenced; + // the max global checkpoint that has been synced + protected volatile long globalCheckpointSynced; + // clean up translog folder uploaded by previous primaries once protected final SetOnce olderPrimaryCleaned = new SetOnce<>(); @@ -437,9 +440,10 @@ private boolean upload(long primaryTerm, long generation, long maxSeqNo) throws config.getNodeId() ).build() ) { + Checkpoint checkpoint = current.getLastSyncedCheckpoint(); return translogTransferManager.transferSnapshot( transferSnapshotProvider, - new RemoteFsTranslogTransferListener(generation, primaryTerm, maxSeqNo) + new RemoteFsTranslogTransferListener(generation, primaryTerm, maxSeqNo, checkpoint.globalCheckpoint) ); } finally { syncPermit.release(SYNC_PERMIT); @@ -474,7 +478,10 @@ public void sync() throws IOException { public boolean syncNeeded() { try (ReleasableLock lock = readLock.acquire()) { return current.syncNeeded() - || (maxRemoteTranslogGenerationUploaded + 1 < this.currentFileGeneration() && current.totalOperations() == 0); + || (maxRemoteTranslogGenerationUploaded + 1 < this.currentFileGeneration() && current.totalOperations() == 0) + // The below condition on GCP exists to handle global checkpoint updates during close index. + // Refer issue - https://github.com/opensearch-project/OpenSearch/issues/15989 + || (current.getLastSyncedCheckpoint().globalCheckpoint > globalCheckpointSynced); } } @@ -674,10 +681,13 @@ private class RemoteFsTranslogTransferListener implements TranslogTransferListen private final long maxSeqNo; - RemoteFsTranslogTransferListener(long generation, long primaryTerm, long maxSeqNo) { + private final long globalCheckpoint; + + RemoteFsTranslogTransferListener(long generation, long primaryTerm, long maxSeqNo, long globalCheckpoint) { this.generation = generation; this.primaryTerm = primaryTerm; this.maxSeqNo = maxSeqNo; + this.globalCheckpoint = globalCheckpoint; } @Override @@ -685,6 +695,11 @@ public void onUploadComplete(TransferSnapshot transferSnapshot) throws IOExcepti maxRemoteTranslogGenerationUploaded = generation; long previousMinRemoteGenReferenced = minRemoteGenReferenced; minRemoteGenReferenced = getMinFileGeneration(); + // Update the global checkpoint only if the supplied global checkpoint is greater than it + // When a new writer is created the + if (globalCheckpoint > globalCheckpointSynced) { + globalCheckpointSynced = globalCheckpoint; + } if (previousMinRemoteGenReferenced != minRemoteGenReferenced) { onMinRemoteGenReferencedChange(); } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index 339d876274557..03c77a9a83f57 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -1801,6 +1801,83 @@ public void testDownloadWithEmptyTranslogOnlyInLocal() throws IOException { assertArrayEquals(filesPostFirstDownload, filesPostSecondDownload); } + public void testSyncWithGlobalCheckpointUpdate() throws IOException { + ArrayList ops = new ArrayList<>(); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 1, primaryTerm.get(), new byte[] { 2 })); + + // Set a global checkpoint + long initialGlobalCheckpoint = 1L; + globalCheckpoint.set(initialGlobalCheckpoint); + + // Sync the translog + translog.sync(); + + // Verify that the globalCheckpointSynced is updated + assertEquals(initialGlobalCheckpoint, ((RemoteFsTranslog) translog).getLastSyncedCheckpoint().globalCheckpoint); + + // Update global checkpoint + long newGlobalCheckpoint = 2L; + globalCheckpoint.set(newGlobalCheckpoint); + + // Add a new operation and sync + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 2, primaryTerm.get(), new byte[] { 3 })); + translog.sync(); + + // Verify that the globalCheckpointSynced is updated to the new value + assertEquals(newGlobalCheckpoint, ((RemoteFsTranslog) translog).getLastSyncedCheckpoint().globalCheckpoint); + } + + public void testSyncNeededWithGlobalCheckpointUpdate() throws IOException { + ArrayList ops = new ArrayList<>(); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); + + // Set initial global checkpoint + long initialGlobalCheckpoint = 0L; + globalCheckpoint.set(initialGlobalCheckpoint); + + // Sync the translog + translog.sync(); + + // Verify that sync is not needed + assertFalse(translog.syncNeeded()); + + // Update global checkpoint + long newGlobalCheckpoint = 1L; + globalCheckpoint.set(newGlobalCheckpoint); + + // Verify that sync is now needed due to global checkpoint update + assertTrue(translog.syncNeeded()); + + // Sync again + translog.sync(); + + // Verify that sync is not needed after syncing + assertFalse(translog.syncNeeded()); + } + + public void testGlobalCheckpointUpdateDuringClose() throws IOException { + ArrayList ops = new ArrayList<>(); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); + + // Set initial global checkpoint + long initialGlobalCheckpoint = 0L; + globalCheckpoint.set(initialGlobalCheckpoint); + + // Sync the translog + translog.sync(); + + // Update global checkpoint + long newGlobalCheckpoint = 1L; + globalCheckpoint.set(newGlobalCheckpoint); + + // Close the translog + translog.close(); + + // Verify that the last synced checkpoint includes the updated global checkpoint + assertEquals(newGlobalCheckpoint, ((RemoteFsTranslog) translog).getLastSyncedCheckpoint().globalCheckpoint); + } + public class ThrowingBlobRepository extends FsRepository { private final Environment environment; From a42e51ddb93d50a9bb0455bb6bda9fe461fe8f5b Mon Sep 17 00:00:00 2001 From: Finn Date: Wed, 25 Sep 2024 16:30:37 -0700 Subject: [PATCH 057/108] Fix segment count for optimized agg rest test (#16067) Signed-off-by: Finn Carroll --- .../search.aggregation/360_date_histogram.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml index 8c8a98b2db22c..0cabbf6f25133 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml @@ -109,6 +109,19 @@ setup: --- "Date histogram aggregation w/ shared field range test": + - do: + indices.create: + index: dhisto-agg-w-query + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + refresh_interval: -1 + mappings: + properties: + date: + type: date + - do: bulk: refresh: true @@ -127,6 +140,11 @@ setup: - '{"index": {}}' - '{"date": "2025-02-14"}' + - do: + indices.forcemerge: + index: dhisto-agg-w-query + max_num_segments: 1 + - do: search: index: dhisto-agg-w-query From daf1669b7ca77c62fc82ba9162d3708702b13fce Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Wed, 25 Sep 2024 18:16:59 -0700 Subject: [PATCH 058/108] Added support for msearch API to pass search pipeline name (#15923) * Added support for search pipeline name in multi search API Signed-off-by: Owais * Updated CHANGELOG Signed-off-by: Owais * Pulled search pipeline in MultiSearchRequest and updated test Signed-off-by: Owais * Updated test Signed-off-by: Owais * Updated SearchRequest with search pipeline from source Signed-off-by: Owais * Added tests for parseSearchRequest Signed-off-by: Owais * Guard serialization with version check Signed-off-by: Owais * Updated version and added another test for serialization Signed-off-by: Owais --------- Signed-off-by: Owais --- CHANGELOG.md | 1 + .../action/search/MultiSearchRequest.java | 4 ++ .../rest/action/search/RestSearchAction.java | 2 +- .../search/builder/SearchSourceBuilder.java | 36 +++++++++++- .../action/search/SearchRequestTests.java | 24 ++++++-- .../builder/SearchSourceBuilderTests.java | 21 +++++++ .../pipeline/SearchPipelineServiceTests.java | 58 +++++++++++++++++++ 7 files changed, 139 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfec81c070bab..935901d410178 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) +- Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java index 5b887b48f696e..f16d7d1e7d6a3 100644 --- a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java @@ -310,6 +310,10 @@ public static void readMultiLineFormat( ) { consumer.accept(searchRequest, parser); } + + if (searchRequest.source() != null && searchRequest.source().pipeline() != null) { + searchRequest.pipeline(searchRequest.source().pipeline()); + } // move pointers from = nextMarker + 1; } diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java index 3a6b45013e892..05465e32631fd 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java @@ -210,7 +210,7 @@ public static void parseSearchRequest( searchRequest.routing(request.param("routing")); searchRequest.preference(request.param("preference")); searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); - searchRequest.pipeline(request.param("search_pipeline")); + searchRequest.pipeline(request.param("search_pipeline", searchRequest.source().pipeline())); checkRestTotalHits(request, searchRequest); request.paramAsBoolean(INCLUDE_NAMED_QUERIES_SCORE_PARAM, false); diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java index 8a9704b04566f..dd4e4d073cb1b 100644 --- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java @@ -224,6 +224,8 @@ public static HighlightBuilder highlight() { private Map searchPipelineSource = null; + private String searchPipeline; + /** * Constructs a new search source builder. */ @@ -297,6 +299,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { derivedFields = in.readList(DerivedField::new); } } + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + searchPipeline = in.readOptionalString(); + } } @Override @@ -377,6 +382,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeList(derivedFields); } } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalString(searchPipeline); + } } /** @@ -1111,6 +1119,13 @@ public Map searchPipelineSource() { return searchPipelineSource; } + /** + * @return a search pipeline name defined within the search source (see {@link org.opensearch.search.pipeline.SearchPipelineService}) + */ + public String pipeline() { + return searchPipeline; + } + /** * Define a search pipeline to process this search request and/or its response. See {@link org.opensearch.search.pipeline.SearchPipelineService}. */ @@ -1119,6 +1134,14 @@ public SearchSourceBuilder searchPipelineSource(Map searchPipeli return this; } + /** + * Define a search pipeline name to process this search request and/or its response. See {@link org.opensearch.search.pipeline.SearchPipelineService}. + */ + public SearchSourceBuilder pipeline(String searchPipeline) { + this.searchPipeline = searchPipeline; + return this; + } + /** * Rewrites this search source builder into its primitive form. e.g. by * rewriting the QueryBuilder. If the builder did not change the identity @@ -1216,6 +1239,7 @@ private SearchSourceBuilder shallowCopy( rewrittenBuilder.pointInTimeBuilder = pointInTimeBuilder; rewrittenBuilder.derivedFieldsObject = derivedFieldsObject; rewrittenBuilder.derivedFields = derivedFields; + rewrittenBuilder.searchPipeline = searchPipeline; return rewrittenBuilder; } @@ -1283,6 +1307,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th sort(parser.text()); } else if (PROFILE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { profile = parser.booleanValue(); + } else if (SEARCH_PIPELINE.match(currentFieldName, parser.getDeprecationHandler())) { + searchPipeline = parser.text(); } else { throw new ParsingException( parser.getTokenLocation(), @@ -1612,6 +1638,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t } + if (searchPipeline != null) { + builder.field(SEARCH_PIPELINE.getPreferredName(), searchPipeline); + } + return builder; } @@ -1889,7 +1919,8 @@ public int hashCode() { trackTotalHitsUpTo, pointInTimeBuilder, derivedFieldsObject, - derivedFields + derivedFields, + searchPipeline ); } @@ -1934,7 +1965,8 @@ public boolean equals(Object obj) { && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo) && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder) && Objects.equals(derivedFieldsObject, other.derivedFieldsObject) - && Objects.equals(derivedFields, other.derivedFields); + && Objects.equals(derivedFields, other.derivedFields) + && Objects.equals(searchPipeline, other.searchPipeline); } @Override diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java index 40514c526f190..acda1445bacbb 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java @@ -42,6 +42,8 @@ import org.opensearch.geometry.LinearRing; import org.opensearch.index.query.GeoShapeQueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.search.RestSearchAction; import org.opensearch.search.AbstractSearchTestCase; import org.opensearch.search.Scroll; import org.opensearch.search.builder.PointInTimeBuilder; @@ -50,14 +52,18 @@ import org.opensearch.search.rescore.QueryRescorerBuilder; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; +import org.opensearch.test.rest.FakeRestRequest; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.function.IntConsumer; import static java.util.Collections.emptyMap; +import static org.opensearch.action.search.SearchType.DFS_QUERY_THEN_FETCH; import static org.opensearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; public class SearchRequestTests extends AbstractSearchTestCase { @@ -242,6 +248,19 @@ public void testCopyConstructor() throws IOException { assertNotSame(deserializedRequest, searchRequest); } + public void testParseSearchRequestWithUnsupportedSearchType() throws IOException { + RestRequest restRequest = new FakeRestRequest(); + SearchRequest searchRequest = createSearchRequest(); + IntConsumer setSize = mock(IntConsumer.class); + restRequest.params().put("search_type", "query_and_fetch"); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> RestSearchAction.parseSearchRequest(searchRequest, restRequest, null, namedWriteableRegistry, setSize) + ); + assertEquals("Unsupported search type [query_and_fetch]", exception.getMessage()); + } + public void testEqualsAndHashcode() throws IOException { checkEqualsAndHashCode(createSearchRequest(), SearchRequest::new, this::mutate); } @@ -268,10 +287,7 @@ private SearchRequest mutate(SearchRequest searchRequest) { ); mutators.add( () -> mutation.searchType( - randomValueOtherThan( - searchRequest.searchType(), - () -> randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH) - ) + randomValueOtherThan(searchRequest.searchType(), () -> randomFrom(DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)) ) ); mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder))); diff --git a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java index 9697f4cee0d58..da8ccc9e121e0 100644 --- a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java @@ -421,6 +421,27 @@ public void testDerivedFieldsParsingAndSerializationObjectType() throws IOExcept } } + public void testSearchPipelineParsingAndSerialization() throws IOException { + String restContent = "{ \"query\": { \"match_all\": {} }, \"from\": 0, \"size\": 10, \"search_pipeline\": \"my_pipeline\" }"; + String expectedContent = "{\"from\":0,\"size\":10,\"query\":{\"match_all\":{\"boost\":1.0}},\"search_pipeline\":\"my_pipeline\"}"; + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser); + searchSourceBuilder = rewrite(searchSourceBuilder); + + try (BytesStreamOutput output = new BytesStreamOutput()) { + searchSourceBuilder.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { + SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in); + String actualContent = deserializedBuilder.toString(); + assertEquals(expectedContent, actualContent); + assertEquals(searchSourceBuilder.hashCode(), deserializedBuilder.hashCode()); + assertNotSame(searchSourceBuilder, deserializedBuilder); + } + } + } + } + public void testAggsParsing() throws IOException { { String restContent = "{\n" diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java index f5857922fdff2..b52205996f34b 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java @@ -969,6 +969,64 @@ public void testInlinePipeline() throws Exception { } } + public void testInlineDefinedPipeline() throws Exception { + SearchPipelineService searchPipelineService = createWithProcessors(); + + SearchPipelineMetadata metadata = new SearchPipelineMetadata( + Map.of( + "p1", + new PipelineConfiguration( + "p1", + new BytesArray( + "{" + + "\"request_processors\": [{ \"scale_request_size\": { \"scale\" : 2 } }]," + + "\"response_processors\": [{ \"fixed_score\": { \"score\" : 2 } }]" + + "}" + ), + MediaTypeRegistry.JSON + ) + ) + + ); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); + ClusterState previousState = clusterState; + clusterState = ClusterState.builder(clusterState) + .metadata(Metadata.builder().putCustom(SearchPipelineMetadata.TYPE, metadata)) + .build(); + searchPipelineService.applyClusterState(new ClusterChangedEvent("", clusterState, previousState)); + + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().size(100).pipeline("p1"); + SearchRequest searchRequest = new SearchRequest().source(sourceBuilder); + searchRequest.pipeline(searchRequest.source().pipeline()); + + // Verify pipeline + PipelinedRequest pipelinedRequest = syncTransformRequest( + searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver) + ); + Pipeline pipeline = pipelinedRequest.getPipeline(); + assertEquals("p1", pipeline.getId()); + assertEquals(1, pipeline.getSearchRequestProcessors().size()); + assertEquals(1, pipeline.getSearchResponseProcessors().size()); + + // Verify that pipeline transforms request + assertEquals(200, pipelinedRequest.source().size()); + + int size = 10; + SearchHit[] hits = new SearchHit[size]; + for (int i = 0; i < size; i++) { + hits[i] = new SearchHit(i, "doc" + i, Collections.emptyMap(), Collections.emptyMap()); + hits[i].score(i); + } + SearchHits searchHits = new SearchHits(hits, new TotalHits(size * 2, TotalHits.Relation.EQUAL_TO), size); + SearchResponseSections searchResponseSections = new SearchResponseSections(searchHits, null, null, false, false, null, 0); + SearchResponse searchResponse = new SearchResponse(searchResponseSections, null, 1, 1, 0, 10, null, null); + + SearchResponse transformedResponse = syncTransformResponse(pipelinedRequest, searchResponse); + for (int i = 0; i < size; i++) { + assertEquals(2.0, transformedResponse.getHits().getHits()[i].getScore(), 0.0001); + } + } + public void testInfo() { SearchPipelineService searchPipelineService = createWithProcessors(); SearchPipelineInfo info = searchPipelineService.info(); From ae22e3ff32ef15a6af302c50872f1fa0e8e140fe Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 25 Sep 2024 22:09:00 -0400 Subject: [PATCH 059/108] Update 390_search_as_you_type.yml (#16065) Signed-off-by: Andriy Redko --- .../rest-api-spec/test/search/390_search_as_you_type.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml index 689b3edfd1066..f29fa3f9aa987 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml @@ -30,8 +30,8 @@ teardown: --- "Test search_as_you_type data type supports multi-fields": - skip: - version: " - 2.99.99" - reason: "the bug was fixed since 3.0.0" + version: " - 2.17.99" + reason: "the bug was fixed since 2.18.0" - do: indices.get_mapping: { From 58888672e648193baaa6bc88ed3730207c3522ff Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Thu, 26 Sep 2024 18:02:43 +0530 Subject: [PATCH 060/108] Allow RemotePublication configured nodes to join remote cluster in mixed mode (#15994) Signed-off-by: Rajiv Kumar Vaidyanathan --- .../RemotePublicationConfigurationIT.java | 273 ++++++++++++++++++ .../MigrationBaseTestCase.java | 4 +- .../coordination/JoinTaskExecutor.java | 15 +- 3 files changed, 287 insertions(+), 5 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java new file mode 100644 index 0000000000000..1d19a4bfd1af5 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java @@ -0,0 +1,273 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.plugins.Plugin; +import org.opensearch.remotemigration.MigrationBaseTestCase; +import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.ReloadableFsRepository; +import org.opensearch.test.InternalSettingsPlugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; +import org.junit.Before; + +import java.util.Collection; +import java.util.Locale; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +/** + * Tests the compatibility between types of nodes based on the configured repositories + * Non Remote node [No Repositories configured] + * Remote Publish Configured Node [Cluster State + Routing Table] + * Remote Node [Cluster State + Segment + Translog] + * Remote Node With Routing Table [Cluster State + Segment + Translog + Routing Table] + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RemotePublicationConfigurationIT extends MigrationBaseTestCase { + private final String REMOTE_PRI_DOCREP_REP = "remote-primary-docrep-replica"; + + @Override + protected Collection> nodePlugins() { + /* Adding the following mock plugins: + - InternalSettingsPlugin : To override default intervals of retention lease and global ckp sync + - MockFsRepositoryPlugin and MockTransportService.TestPlugin: To ensure remote interactions are not no-op and retention leases are properly propagated + */ + return Stream.concat( + super.nodePlugins().stream(), + Stream.of(InternalSettingsPlugin.class, MockFsRepositoryPlugin.class, MockTransportService.TestPlugin.class) + ).collect(Collectors.toList()); + } + + @Before + public void setUp() throws Exception { + if (segmentRepoPath == null || translogRepoPath == null) { + segmentRepoPath = randomRepoPath().toAbsolutePath(); + translogRepoPath = randomRepoPath().toAbsolutePath(); + } + super.setUp(); + } + + public Settings.Builder remotePublishConfiguredNodeSetting() { + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_NAME + ); + String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey(); + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + REPOSITORY_NAME + ); + String routingTableRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + ROUTING_TABLE_REPO_NAME + ); + String routingTableRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + ROUTING_TABLE_REPO_NAME + ); + + Settings.Builder builder = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) + .put(stateRepoTypeAttributeKey, ReloadableFsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) + .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, true) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO_NAME) + .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE) + .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath); + return builder; + } + + public Settings.Builder remoteWithRoutingTableNodeSetting() { + // Remote Cluster with Routing table + return Settings.builder() + .put( + buildRemoteStoreNodeAttributes( + REPOSITORY_NAME, + segmentRepoPath, + REPOSITORY_2_NAME, + translogRepoPath, + REPOSITORY_NAME, + segmentRepoPath, + false + ) + ) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true); + } + + public void testRemotePublishConfigNodeJoinNonRemoteCluster() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ensureStableCluster(6); + ensureGreen(); + } + + public void testRemotePublishConfigNodeJoinRemoteCluster() throws Exception { + // Remote Cluster without Routing table + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + setAddRemote(false); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeWithRoutingTableJoinRemoteCluster() throws Exception { + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + setAddRemote(false); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + // Remote Repo with Routing table + Settings settings = remoteWithRoutingTableNodeSetting().build(); + internalCluster().startClusterManagerOnlyNode(settings); + ensureStableCluster(4); + ensureGreen(); + } + + public void testNonRemoteNodeJoinRemoteWithRoutingCluster() throws Exception { + Settings settings = remoteWithRoutingTableNodeSetting().build(); + internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(2, settings); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + internalCluster().startClusterManagerOnlyNode(); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemotePublishConfigNodeJoinRemoteWithRoutingCluster() throws Exception { + Settings settings = remoteWithRoutingTableNodeSetting().build(); + internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(2, settings); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + internalCluster().startClusterManagerOnlyNode(remotePublishConfiguredNodeSetting().build()); + + ensureStableCluster(4); + ensureGreen(); + } + + public void testNonRemoteNodeJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + internalCluster().startClusterManagerOnlyNode(); + + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeWithRoutingTableJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + + Settings settings = Settings.builder() + .put( + buildRemoteStoreNodeAttributes( + REPOSITORY_NAME, + segmentRepoPath, + REPOSITORY_2_NAME, + translogRepoPath, + ROUTING_TABLE_REPO_NAME, + segmentRepoPath, + false + ) + ) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + internalCluster().startClusterManagerOnlyNode(settings); + + ensureStableCluster(4); + ensureGreen(); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java index 17a9c3ddbe317..a82e6d45ce0f6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java @@ -57,6 +57,8 @@ public class MigrationBaseTestCase extends OpenSearchIntegTestCase { protected static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected static final String ROUTING_TABLE_REPO_NAME = "remote-routing-repo"; + protected static final String REPOSITORY_2_NAME = "test-remote-store-repo-2"; protected Path segmentRepoPath; @@ -72,7 +74,7 @@ public class MigrationBaseTestCase extends OpenSearchIntegTestCase { randomAlphaOfLength(5) ); - void setAddRemote(boolean addRemote) { + public void setAddRemote(boolean addRemote) { this.addRemote = addRemote; } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 838b5723b217b..13033b670d44b 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -420,9 +420,14 @@ public static void ensureIndexCompatibility(final Version nodeVersion, Metadata * ensures that the joining node has a version that's compatible with all current nodes */ public static void ensureNodesCompatibility(final DiscoveryNode joiningNode, DiscoveryNodes currentNodes, Metadata metadata) { - final Version minNodeVersion = currentNodes.getMinNodeVersion(); - final Version maxNodeVersion = currentNodes.getMaxNodeVersion(); - ensureNodesCompatibility(joiningNode, currentNodes, metadata, minNodeVersion, maxNodeVersion); + try { + final Version minNodeVersion = currentNodes.getMinNodeVersion(); + final Version maxNodeVersion = currentNodes.getMaxNodeVersion(); + ensureNodesCompatibility(joiningNode, currentNodes, metadata, minNodeVersion, maxNodeVersion); + } catch (Exception e) { + logger.error("Exception in NodesCompatibility validation", e); + throw e; + } } /** @@ -539,9 +544,11 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); List reposToSkip = new ArrayList<>(1); + // find a remote node which has routing table configured Optional remoteRoutingTableNode = existingNodes.stream() .filter( - node -> node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) != null + node -> node.isRemoteStoreNode() + && node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) != null ) .findFirst(); // If none of the existing nodes have routing table repo, then we skip this repo check if present in joining node. From 949b31f986c559d90017bef5d5d36455de75a77a Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 26 Sep 2024 11:47:20 -0400 Subject: [PATCH 061/108] Ignore google-cloud-storage and google-api-client major version upgrade for dependabot (#16072) * Ignore google-cloud-storage major version upgrade for dependabot Signed-off-by: Craig Perkins * Include google-api-client Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8c4f4d59ea1fc..af83f8b2edac2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -722,6 +722,12 @@ updates: - directory: /plugins/ open-pull-requests-limit: 1 package-ecosystem: gradle + ignore: + # For all packages, ignore all major versions to minimize breaking issues + - dependency-name: "com.google.cloud:google-cloud-storage" + update-types: [ "version-update:semver-major" ] + - dependency-name: "com.google.api-client:google-api-client" + update-types: [ "version-update:semver-major" ] schedule: interval: weekly labels: From 007d1b97a010d2b7ec15f257c4c9c65d61a789c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 13:54:51 -0400 Subject: [PATCH 062/108] Bump com.nimbusds:nimbus-jose-jwt from 9.40 to 9.41.1 in /plugins/repository-azure (#16038) * Bump com.nimbusds:nimbus-jose-jwt in /plugins/repository-azure Bumps [com.nimbusds:nimbus-jose-jwt](https://bitbucket.org/connect2id/nimbus-jose-jwt) from 9.40 to 9.41.1. - [Changelog](https://bitbucket.org/connect2id/nimbus-jose-jwt/src/master/CHANGELOG.txt) - [Commits](https://bitbucket.org/connect2id/nimbus-jose-jwt/branches/compare/9.41.1..9.40) --- updated-dependencies: - dependency-name: com.nimbusds:nimbus-jose-jwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 | 1 - .../repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 create mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 935901d410178..e88f85453109e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) +- Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038)) - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 616da0be62c30..2892bdba51ba6 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -63,7 +63,7 @@ dependencies { api "net.java.dev.jna:jna-platform:${versions.jna}" api 'com.microsoft.azure:msal4j:1.17.1' api 'com.nimbusds:oauth2-oidc-sdk:11.19.1' - api 'com.nimbusds:nimbus-jose-jwt:9.40' + api 'com.nimbusds:nimbus-jose-jwt:9.41.1' api 'com.nimbusds:content-type:2.3' api 'com.nimbusds:lang-tag:1.7' // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart, diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 deleted file mode 100644 index 83228caf233cc..0000000000000 --- a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -42b1dfa0360e4062951b070bac52dd8d96fd7b38 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 new file mode 100644 index 0000000000000..71fa950cb9530 --- /dev/null +++ b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 @@ -0,0 +1 @@ +35532a88e1b49a623ec97fd276cc208ea525b6bc \ No newline at end of file From 7caca26fcb2dfee9e28ad09cc2de0e5d0248dba9 Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Thu, 26 Sep 2024 16:09:07 -0700 Subject: [PATCH 063/108] Updates version to fix BWC for SearchSourceBuilder (#16086) Signed-off-by: Owais --- .../org/opensearch/search/builder/SearchSourceBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java index dd4e4d073cb1b..9c438401b9fbe 100644 --- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java @@ -299,7 +299,7 @@ public SearchSourceBuilder(StreamInput in) throws IOException { derivedFields = in.readList(DerivedField::new); } } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { searchPipeline = in.readOptionalString(); } } @@ -382,7 +382,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeList(derivedFields); } } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeOptionalString(searchPipeline); } } From e8b02c9545d7908ae1a3ea2d109c06ff3e0c07a5 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:02:24 +0530 Subject: [PATCH 064/108] Shallow snapshot v2 - create snapshot validations in a cluster state update (#15939) --------- Signed-off-by: Gaurav Bafna --- .../remotestore/RemoteRestoreSnapshotIT.java | 126 +++-- .../remotestore/RemoteSnapshotIT.java | 89 ++++ .../snapshots/CloneSnapshotV2IT.java | 106 ++-- .../snapshots/ConcurrentSnapshotsV2IT.java | 486 ++++++++++++++++++ .../cluster/SnapshotsInProgress.java | 141 ++++- .../snapshots/SnapshotsService.java | 410 ++++++++++----- ...SnapshotsInProgressSerializationTests.java | 6 +- 7 files changed, 1134 insertions(+), 230 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 0acb578e2e7bf..a0183e89bfce2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -11,6 +11,8 @@ import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; @@ -25,6 +27,7 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.index.Index; @@ -43,14 +46,11 @@ import org.opensearch.repositories.RepositoryData; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.FsRepository; -import org.opensearch.snapshots.AbstractSnapshotIntegTestCase; import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.snapshots.SnapshotRestoreException; import org.opensearch.snapshots.SnapshotState; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; -import org.junit.After; -import org.junit.Before; import java.io.IOException; import java.nio.file.Files; @@ -63,6 +63,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -79,48 +80,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class RemoteRestoreSnapshotIT extends AbstractSnapshotIntegTestCase { - private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; - private Path remoteRepoPath; - - @Before - public void setup() { - remoteRepoPath = randomRepoPath().toAbsolutePath(); - } - - @After - public void teardown() { - clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get(); - } - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath)) - .build(); - } - - private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { - Settings.Builder settingsBuilder = Settings.builder() - .put(super.indexSettings()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) - .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s"); - return settingsBuilder; - } - - private void indexDocuments(Client client, String indexName, int numOfDocs) { - indexDocuments(client, indexName, 0, numOfDocs); - } - - private void indexDocuments(Client client, String indexName, int fromId, int toId) { - for (int i = fromId; i < toId; i++) { - String id = Integer.toString(i); - client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); - } - client.admin().indices().prepareFlush(indexName).get(); - } +public class RemoteRestoreSnapshotIT extends RemoteSnapshotIT { private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) { for (int i = 0; i < numOfDocs; i++) { @@ -997,6 +957,75 @@ public void testConcurrentSnapshotV2CreateOperation() throws InterruptedExceptio assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1)); } + public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Exception { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(snapshotRepoName, FsRepository.TYPE, settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + Thread thread = new Thread(() -> { + try { + String snapshotName = "snapshot-earlier-master"; + internalCluster().nonClusterManagerClient() + .admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName) + .setWaitForCompletion(true) + .setMasterNodeTimeout(TimeValue.timeValueSeconds(60)) + .get(); + + } catch (Exception ignored) {} + }); + thread.start(); + + // stop existing master + final String clusterManagerNode = internalCluster().getClusterManagerName(); + stopNode(clusterManagerNode); + + // Validate that we have greater one snapshot has been created + String snapshotName = "new-snapshot"; + try { + client().admin().cluster().prepareCreateSnapshot(snapshotRepoName, snapshotName).setWaitForCompletion(true).get(); + } catch (Exception e) { + logger.info("Exception while creating new-snapshot", e); + } + + // Validate that snapshot is present in repository data + assertBusy(() -> { + GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName); + GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet(); + assertThat(response2.getSnapshots().size(), greaterThanOrEqualTo(1)); + }, 30, TimeUnit.SECONDS); + thread.join(); + } + public void testCreateSnapshotV2WithRedIndex() throws Exception { internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); internalCluster().startDataOnlyNode(pinnedTimestampSettings()); @@ -1315,11 +1344,4 @@ public void testConcurrentV1SnapshotAndV2RepoSettingUpdate() throws Exception { createV1SnapshotThread.join(); } - private Settings pinnedTimestampSettings() { - Settings settings = Settings.builder() - .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) - .build(); - return settings; - } - } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java new file mode 100644 index 0000000000000..836871b8251d1 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore; + +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.repositories.fs.ReloadableFsRepository; +import org.opensearch.snapshots.AbstractSnapshotIntegTestCase; +import org.junit.After; +import org.junit.Before; + +import java.nio.file.Path; +import java.util.concurrent.ExecutionException; + +import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING; + +public abstract class RemoteSnapshotIT extends AbstractSnapshotIntegTestCase { + protected static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; + protected Path remoteRepoPath; + + @Before + public void setup() { + remoteRepoPath = randomRepoPath().toAbsolutePath(); + } + + @After + public void teardown() { + clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath)) + .build(); + } + + protected Settings pinnedTimestampSettings() { + Settings settings = Settings.builder() + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .build(); + return settings; + } + + protected Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { + Settings.Builder settingsBuilder = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s"); + return settingsBuilder; + } + + protected void indexDocuments(Client client, String indexName, int numOfDocs) { + indexDocuments(client, indexName, 0, numOfDocs); + } + + void indexDocuments(Client client, String indexName, int fromId, int toId) { + for (int i = fromId; i < toId; i++) { + String id = Integer.toString(i); + client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); + } + client.admin().indices().prepareFlush(indexName).get(); + } + + protected void setFailRate(String repoName, int value) throws ExecutionException, InterruptedException { + GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { repoName }); + GetRepositoriesResponse res = client().admin().cluster().getRepositories(gr).get(); + RepositoryMetadata rmd = res.repositories().get(0); + Settings.Builder settings = Settings.builder() + .put("location", rmd.settings().get("location")) + .put(REPOSITORIES_FAILRATE_SETTING.getKey(), value); + createRepository(repoName, ReloadableFsRepository.TYPE, settings); + } + +} diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java index c6744ae62db60..69e85b13548e0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java @@ -34,11 +34,15 @@ import org.opensearch.action.ActionRunnable; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.rest.RestStatus; @@ -50,8 +54,10 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.nio.file.Path; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; -import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -134,27 +140,32 @@ public void testCloneShallowCopyV2() throws Exception { assertTrue(response.isAcknowledged()); awaitClusterManagerFinishRepoOperations(); + AtomicReference cloneSnapshotId = new AtomicReference<>(); // Validate that snapshot is present in repository data - PlainActionFuture repositoryDataPlainActionFutureClone = new PlainActionFuture<>(); - repository.getRepositoryData(repositoryDataPlainActionFutureClone); - - repositoryData = repositoryDataPlainActionFutureClone.get(); - assertEquals(repositoryData.getSnapshotIds().size(), 2); - boolean foundCloneInRepoData = false; - SnapshotId cloneSnapshotId = null; - for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) { - if (snapshotId.getName().equals("test_clone_snapshot1")) { - foundCloneInRepoData = true; - cloneSnapshotId = snapshotId; + waitUntil(() -> { + PlainActionFuture repositoryDataPlainActionFutureClone = new PlainActionFuture<>(); + repository.getRepositoryData(repositoryDataPlainActionFutureClone); + + RepositoryData repositoryData1; + try { + repositoryData1 = repositoryDataPlainActionFutureClone.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); } - } - final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId; + for (SnapshotId snapshotId : repositoryData1.getSnapshotIds()) { + if (snapshotId.getName().equals("test_clone_snapshot1")) { + cloneSnapshotId.set(snapshotId); + return true; + } + } + return false; + }, 90, TimeUnit.SECONDS); + + final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId.get(); SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get( f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal))) ); - assertTrue(foundCloneInRepoData); - assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp())); for (String index : sourceSnapshotInfo.indices()) { assertTrue(cloneSnapshotInfo.indices().contains(index)); @@ -259,19 +270,21 @@ public void testCloneShallowCopyAfterDisablingV2() throws Exception { assertThat(sourceSnapshotInfoV1.state(), equalTo(SnapshotState.SUCCESS)); assertThat(sourceSnapshotInfoV1.successfulShards(), greaterThan(0)); assertThat(sourceSnapshotInfoV1.successfulShards(), equalTo(sourceSnapshotInfoV1.totalShards())); - assertThat(sourceSnapshotInfoV1.getPinnedTimestamp(), equalTo(0L)); + // assertThat(sourceSnapshotInfoV1.getPinnedTimestamp(), equalTo(0L)); + AtomicReference repositoryDataAtomicReference = new AtomicReference<>(); + awaitClusterManagerFinishRepoOperations(); // Validate that snapshot is present in repository data - PlainActionFuture repositoryDataV1PlainActionFuture = new PlainActionFuture<>(); - BlobStoreRepository repositoryV1 = (BlobStoreRepository) internalCluster().getCurrentClusterManagerNodeInstance( - RepositoriesService.class - ).repository(snapshotRepoName); - repositoryV1.getRepositoryData(repositoryDataV1PlainActionFuture); - - repositoryData = repositoryDataV1PlainActionFuture.get(); + assertBusy(() -> { + Metadata metadata = clusterAdmin().prepareState().get().getState().metadata(); + RepositoriesMetadata repositoriesMetadata = metadata.custom(RepositoriesMetadata.TYPE); + assertEquals(1, repositoriesMetadata.repository(snapshotRepoName).generation()); + assertEquals(1, repositoriesMetadata.repository(snapshotRepoName).pendingGeneration()); - assertTrue(repositoryData.getSnapshotIds().contains(sourceSnapshotInfoV1.snapshotId())); - assertEquals(repositoryData.getSnapshotIds().size(), 2); + GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName); + GetSnapshotsResponse response = client().admin().cluster().getSnapshots(request).actionGet(); + assertEquals(2, response.getSnapshots().size()); + }, 30, TimeUnit.SECONDS); // clone should get created for v2 snapshot AcknowledgedResponse response = client().admin() @@ -289,31 +302,28 @@ public void testCloneShallowCopyAfterDisablingV2() throws Exception { ).repository(snapshotRepoName); repositoryCloneV2.getRepositoryData(repositoryDataCloneV2PlainActionFuture); - repositoryData = repositoryDataCloneV2PlainActionFuture.get(); - - assertEquals(repositoryData.getSnapshotIds().size(), 3); - boolean foundCloneInRepoData = false; - SnapshotId cloneSnapshotId = null; - for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) { - if (snapshotId.getName().equals(cloneSnapshotV2)) { - foundCloneInRepoData = true; - cloneSnapshotId = snapshotId; - } - } - final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId; - SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get( - f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal))) - ); + // Validate that snapshot is present in repository data + assertBusy(() -> { + Metadata metadata = clusterAdmin().prepareState().get().getState().metadata(); + RepositoriesMetadata repositoriesMetadata = metadata.custom(RepositoriesMetadata.TYPE); + assertEquals(2, repositoriesMetadata.repository(snapshotRepoName).generation()); + assertEquals(2, repositoriesMetadata.repository(snapshotRepoName).pendingGeneration()); + GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName); + GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet(); + assertEquals(3, response2.getSnapshots().size()); + }, 30, TimeUnit.SECONDS); - assertTrue(foundCloneInRepoData); // pinned timestamp value in clone snapshot v2 matches source snapshot v2 - assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp())); - for (String index : sourceSnapshotInfo.indices()) { - assertTrue(cloneSnapshotInfo.indices().contains(index)); - + GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName, new String[] { sourceSnapshotV2, cloneSnapshotV2 }); + GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet(); + + SnapshotInfo sourceInfo = response2.getSnapshots().get(0); + SnapshotInfo cloneInfo = response2.getSnapshots().get(1); + assertEquals(sourceInfo.getPinnedTimestamp(), cloneInfo.getPinnedTimestamp()); + assertEquals(sourceInfo.totalShards(), cloneInfo.totalShards()); + for (String index : sourceInfo.indices()) { + assertTrue(cloneInfo.indices().contains(index)); } - assertThat(cloneSnapshotInfo.totalShards(), equalTo(sourceSnapshotInfo.totalShards())); - } public void testRestoreFromClone() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java new file mode 100644 index 0000000000000..f20fddb6af26c --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java @@ -0,0 +1,486 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.snapshots; + +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.remotestore.RemoteSnapshotIT; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; +import org.opensearch.repositories.RepositoryData; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class ConcurrentSnapshotsV2IT extends RemoteSnapshotIT { + + public void testLongRunningSnapshotDontAllowConcurrentSnapshot() throws Exception { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + blockClusterManagerOnWriteIndexFile(repoName); + + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-queued"); + awaitNumberOfSnapshotsInProgress(1); + + try { + String snapshotName = "snapshot-concurrent"; + client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName).setWaitForCompletion(true).get(); + fail(); + } catch (Exception e) {} + + unblockNode(repoName, clusterManagerName); + CreateSnapshotResponse csr = snapshotFuture.actionGet(); + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(1, snapInfo.size()); + assertThat(snapInfo, contains(csr.getSnapshotInfo())); + } + + public void testCreateSnapshotFailInFinalize() throws Exception { + final String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-queued"); + awaitNumberOfSnapshotsInProgress(1); + waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L)); + unblockNode(repoName, clusterManagerNode); + expectThrows(SnapshotException.class, snapshotFuture::actionGet); + + final ActionFuture snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success"); + // Second create works out cleanly since the repo + CreateSnapshotResponse csr = snapshotFuture2.actionGet(); + + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(1, snapInfo.size()); + assertThat(snapInfo, contains(csr.getSnapshotInfo())); + } + + public void testCreateSnapshotV2MasterSwitch() throws Exception { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + String clusterManagerNode = internalCluster().getClusterManagerName(); + + blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-queued"); + awaitNumberOfSnapshotsInProgress(1); + waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L)); + + // Fail the cluster manager + stopNode(clusterManagerNode); + + ensureGreen(); + + final ActionFuture snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success"); + // Second create works out cleanly since the repo + CreateSnapshotResponse csr = snapshotFuture2.actionGet(); + + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(1, snapInfo.size()); + assertThat(snapInfo, contains(csr.getSnapshotInfo())); + + } + + public void testPinnedTimestampFailSnapshot() throws Exception { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + // fail segment repo - this is to fail the timestamp pinning + setFailRate(BASE_REMOTE_REPO, 100); + + try { + String snapshotName = "snapshot-fail"; + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .get(); + fail(); + } catch (Exception e) {} + + setFailRate(BASE_REMOTE_REPO, 0); + String snapshotName = "snapshot-success"; + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .get(); + + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(1, snapInfo.size()); + } + + public void testConcurrentSnapshotV2CreateOperation() throws InterruptedException, ExecutionException { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(snapshotRepoName, FsRepository.TYPE, settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + int concurrentSnapshots = 5; + + // Prepare threads for concurrent snapshot creation + List threads = new ArrayList<>(); + + for (int i = 0; i < concurrentSnapshots; i++) { + int snapshotIndex = i; + Thread thread = new Thread(() -> { + try { + String snapshotName = "snapshot-concurrent-" + snapshotIndex; + CreateSnapshotResponse createSnapshotResponse2 = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName) + .setWaitForCompletion(true) + .get(); + SnapshotInfo snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName)); + assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L)); + } catch (Exception e) {} + }); + threads.add(thread); + } + // start all threads + for (Thread thread : threads) { + thread.start(); + } + + // Wait for all threads to complete + for (Thread thread : threads) { + thread.join(); + } + + // Validate that only one snapshot has been created + Repository repository = internalCluster().getInstance(RepositoriesService.class).repository(snapshotRepoName); + PlainActionFuture repositoryDataPlainActionFuture = new PlainActionFuture<>(); + repository.getRepositoryData(repositoryDataPlainActionFuture); + + RepositoryData repositoryData = repositoryDataPlainActionFuture.get(); + assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1)); + } + + public void testLongRunningSnapshotDontAllowConcurrentClone() throws Exception { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + String sourceSnap = "snapshot-source"; + + final CreateSnapshotResponse csr = startFullSnapshot(repoName, sourceSnap).actionGet(); + blockClusterManagerOnWriteIndexFile(repoName); + + final ActionFuture snapshotFuture = startCloneSnapshot(repoName, sourceSnap, "snapshot-clone"); + awaitNumberOfSnapshotsInProgress(1); + + final ActionFuture snapshotFuture2 = startCloneSnapshot(repoName, sourceSnap, "snapshot-clone-2"); + assertThrows(ConcurrentSnapshotExecutionException.class, snapshotFuture2::actionGet); + + unblockNode(repoName, clusterManagerName); + assertThrows(SnapshotException.class, snapshotFuture2::actionGet); + + snapshotFuture.get(); + + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(2, snapInfo.size()); + } + + public void testCloneSnapshotFailInFinalize() throws Exception { + final String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + String sourceSnap = "snapshot-source"; + CreateSnapshotResponse sourceResp = startFullSnapshot(repoName, sourceSnap).actionGet(); + + blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture snapshotFuture = startCloneSnapshot(repoName, sourceSnap, "snapshot-queued"); + awaitNumberOfSnapshotsInProgress(1); + waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L)); + unblockNode(repoName, clusterManagerNode); + assertThrows(SnapshotException.class, snapshotFuture::actionGet); + + final ActionFuture snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success"); + // Second create works out cleanly since the repo is cleaned up + CreateSnapshotResponse csr = snapshotFuture2.actionGet(); + + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(2, snapInfo.size()); + assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), sourceResp.getSnapshotInfo())); + } + + public void testCloneSnapshotV2MasterSwitch() throws Exception { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + String sourceSnap = "snapshot-source"; + CreateSnapshotResponse csr = startFullSnapshot(repoName, sourceSnap).actionGet(); + + String clusterManagerNode = internalCluster().getClusterManagerName(); + + blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture snapshotFuture = startCloneSnapshot(repoName, sourceSnap, "snapshot-queued"); + awaitNumberOfSnapshotsInProgress(1); + waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L)); + + // Fail the cluster manager + stopNode(clusterManagerNode); + + ensureGreen(); + + final ActionFuture snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success"); + // Second create works out cleanly since the repo + CreateSnapshotResponse csr2 = snapshotFuture2.actionGet(); + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(2, snapInfo.size()); + assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), csr2.getSnapshotInfo())); + } + + protected ActionFuture startCloneSnapshot(String repoName, String sourceSnapshotName, String snapshotName) { + logger.info("--> creating full snapshot [{}] to repo [{}]", snapshotName, repoName); + return clusterAdmin().prepareCloneSnapshot(repoName, sourceSnapshotName, snapshotName).setIndices("*").execute(); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index d658f38430dd9..595f6d54fd9be 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -134,6 +134,38 @@ public static Entry startedEntry( ); } + public static Entry startedEntry( + Snapshot snapshot, + boolean includeGlobalState, + boolean partial, + List indices, + List dataStreams, + long startTime, + long repositoryStateId, + final Map shards, + Map userMetadata, + Version version, + boolean remoteStoreIndexShallowCopy, + boolean remoteStoreIndexShallowCopyV2 + ) { + return new SnapshotsInProgress.Entry( + snapshot, + includeGlobalState, + partial, + completed(shards.values()) ? State.SUCCESS : State.STARTED, + indices, + dataStreams, + startTime, + repositoryStateId, + shards, + null, + userMetadata, + version, + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 + ); + } + /** * Creates the initial snapshot clone entry * @@ -168,8 +200,39 @@ public static Entry startClone( version, source, Map.of(), - false // initialising to false, will be updated in startCloning method of SnapshotsService while updating entry with - // clone jobs + false, + false// initialising to false, will be updated in startCloning method of SnapshotsService while updating entry with + // clone jobs + ); + } + + public static Entry startClone( + Snapshot snapshot, + SnapshotId source, + List indices, + long startTime, + long repositoryStateId, + Version version, + boolean remoteStoreIndexShallowCopyV2 + ) { + return new SnapshotsInProgress.Entry( + snapshot, + true, + false, + State.STARTED, + indices, + Collections.emptyList(), + startTime, + repositoryStateId, + Map.of(), + null, + Collections.emptyMap(), + version, + source, + Map.of(), + remoteStoreIndexShallowCopyV2, + remoteStoreIndexShallowCopyV2// initialising to false, will be updated in startCloning method of SnapshotsService + // while updating entry with clone jobs ); } @@ -183,6 +246,8 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation private final Snapshot snapshot; private final boolean includeGlobalState; private final boolean remoteStoreIndexShallowCopy; + + private final boolean remoteStoreIndexShallowCopyV2; private final boolean partial; /** * Map of {@link ShardId} to {@link ShardSnapshotStatus} tracking the state of each shard snapshot operation. @@ -212,6 +277,42 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation @Nullable private final String failure; + public Entry( + Snapshot snapshot, + boolean includeGlobalState, + boolean partial, + State state, + List indices, + List dataStreams, + long startTime, + long repositoryStateId, + final Map shards, + String failure, + Map userMetadata, + Version version, + boolean remoteStoreIndexShallowCopy, + boolean remoteStoreIndexShallowCopyV2 + ) { + this( + snapshot, + includeGlobalState, + partial, + state, + indices, + dataStreams, + startTime, + repositoryStateId, + shards, + failure, + userMetadata, + version, + null, + Map.of(), + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 + ); + } + // visible for testing, use #startedEntry and copy constructors in production code public Entry( Snapshot snapshot, @@ -243,7 +344,8 @@ public Entry( version, null, Map.of(), - remoteStoreIndexShallowCopy + remoteStoreIndexShallowCopy, + false ); } @@ -262,7 +364,8 @@ private Entry( Version version, @Nullable SnapshotId source, @Nullable final Map clones, - boolean remoteStoreIndexShallowCopy + boolean remoteStoreIndexShallowCopy, + boolean remoteStoreIndexShallowCopyV2 ) { this.state = state; this.snapshot = snapshot; @@ -284,7 +387,9 @@ private Entry( this.clones = Collections.unmodifiableMap(clones); } this.remoteStoreIndexShallowCopy = remoteStoreIndexShallowCopy; - assert assertShardsConsistent(this.source, this.state, this.indices, this.shards, this.clones); + this.remoteStoreIndexShallowCopyV2 = remoteStoreIndexShallowCopyV2; + assert this.remoteStoreIndexShallowCopyV2 + || assertShardsConsistent(this.source, this.state, this.indices, this.shards, this.clones); } private Entry(StreamInput in) throws IOException { @@ -307,6 +412,11 @@ private Entry(StreamInput in) throws IOException { } else { remoteStoreIndexShallowCopy = false; } + if (in.getVersion().onOrAfter(Version.CURRENT)) { + remoteStoreIndexShallowCopyV2 = in.readBoolean(); + } else { + remoteStoreIndexShallowCopyV2 = false; + } } private static boolean assertShardsConsistent( @@ -428,7 +538,8 @@ public Entry withRepoGen(long newRepoGen) { version, source, clones, - remoteStoreIndexShallowCopy + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 ); } @@ -451,7 +562,8 @@ public Entry withClones(final Map update version, source, updatedClones, - remoteStoreIndexShallowCopy + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 ); } @@ -471,7 +583,8 @@ public Entry withRemoteStoreIndexShallowCopy(final boolean remoteStoreIndexShall version, source, clones, - remoteStoreIndexShallowCopy + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 ); } @@ -527,7 +640,8 @@ public Entry fail(final Map shards, State state, S version, source, clones, - remoteStoreIndexShallowCopy + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 ); } @@ -614,6 +728,10 @@ public boolean remoteStoreIndexShallowCopy() { return remoteStoreIndexShallowCopy; } + public boolean remoteStoreIndexShallowCopyV2() { + return remoteStoreIndexShallowCopyV2; + } + public Map userMetadata() { return userMetadata; } @@ -678,6 +796,7 @@ public boolean equals(Object o) { if (Objects.equals(source, ((Entry) o).source) == false) return false; if (clones.equals(((Entry) o).clones) == false) return false; if (remoteStoreIndexShallowCopy != entry.remoteStoreIndexShallowCopy) return false; + if (remoteStoreIndexShallowCopyV2 != entry.remoteStoreIndexShallowCopyV2) return false; return true; } @@ -695,6 +814,7 @@ public int hashCode() { result = 31 * result + (source == null ? 0 : source.hashCode()); result = 31 * result + clones.hashCode(); result = 31 * result + (remoteStoreIndexShallowCopy ? 1 : 0); + result = 31 * result + (remoteStoreIndexShallowCopyV2 ? 1 : 0); return result; } @@ -766,6 +886,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeBoolean(remoteStoreIndexShallowCopy); } + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeBoolean(remoteStoreIndexShallowCopyV2); + } } @Override diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 23f6deff3715d..22b2a72b36026 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -468,26 +468,33 @@ public void createSnapshotV2(final CreateSnapshotRequest request, final ActionLi long pinnedTimestamp = System.currentTimeMillis(); final String repositoryName = request.repository(); final String snapshotName = indexNameExpressionResolver.resolveDateMathExpression(request.snapshot()); - validate(repositoryName, snapshotName); - final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot Repository repository = repositoriesService.repository(repositoryName); + validate(repositoryName, snapshotName); + repository.executeConsistentStateUpdate(repositoryData -> new ClusterStateUpdateTask(Priority.URGENT) { + private SnapshotsInProgress.Entry newEntry; - if (repository.isReadOnly()) { - listener.onFailure( - new RepositoryException(repository.getMetadata().name(), "cannot create snapshot-v2 in a readonly repository") - ); - return; - } + private SnapshotId snapshotId; - final Snapshot snapshot = new Snapshot(repositoryName, snapshotId); - ClusterState currentState = clusterService.state(); - final Map userMeta = repository.adaptUserMetadata(request.userMetadata()); - try { - final StepListener repositoryDataListener = new StepListener<>(); - repositoriesService.getRepositoryData(repositoryName, repositoryDataListener); + private Snapshot snapshot; + + boolean enteredLoop; + + @Override + public ClusterState execute(ClusterState currentState) { + // move to in progress + snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot + Repository repository = repositoriesService.repository(repositoryName); + + if (repository.isReadOnly()) { + listener.onFailure( + new RepositoryException(repository.getMetadata().name(), "cannot create snapshot-v2 in a readonly repository") + ); + } + + snapshot = new Snapshot(repositoryName, snapshotId); + final Map userMeta = repository.adaptUserMetadata(request.userMetadata()); - repositoryDataListener.whenComplete(repositoryData -> { createSnapshotPreValidations(currentState, repositoryData, repositoryName, snapshotName); List indices = new ArrayList<>(currentState.metadata().indices().keySet()); @@ -498,7 +505,7 @@ public void createSnapshotV2(final CreateSnapshotRequest request, final ActionLi request.indices() ); - logger.trace("[{}][{}] creating snapshot-v2 for indices [{}]", repositoryName, snapshotName, indices); + logger.info("[{}][{}] creating snapshot-v2 for indices [{}]", repositoryName, snapshotName, indices); final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); final List runningSnapshots = snapshots.entries(); @@ -509,74 +516,139 @@ public void createSnapshotV2(final CreateSnapshotRequest request, final ActionLi IndexId.DEFAULT_SHARD_PATH_TYPE ); final Version version = minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null); - final ShardGenerations shardGenerations = buildShardsGenerationFromRepositoryData( - currentState.metadata(), - currentState.routingTable(), - indexIds, - repositoryData - ); if (repositoryData.getGenId() == RepositoryData.UNKNOWN_REPO_GEN) { logger.debug("[{}] was aborted before starting", snapshot); throw new SnapshotException(snapshot, "Aborted on initialization"); } + + Map shards = new HashMap<>(); + + newEntry = SnapshotsInProgress.startedEntry( + new Snapshot(repositoryName, snapshotId), + request.includeGlobalState(), + request.partial(), + indexIds, + dataStreams, + threadPool.absoluteTimeInMillis(), + repositoryData.getGenId(), + shards, + userMeta, + version, + true, + true + ); + final List newEntries = new ArrayList<>(runningSnapshots); + newEntries.add(newEntry); + + // Entering finalize loop here to prevent concurrent snapshots v2 snapshots + enteredLoop = tryEnterRepoLoop(repositoryName); + if (enteredLoop == false) { + throw new ConcurrentSnapshotExecutionException( + repositoryName, + snapshotName, + "cannot start snapshot-v2 while a repository is in finalization state" + ); + } + return ClusterState.builder(currentState) + .putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(new ArrayList<>(newEntries))) + .build(); + } + + @Override + public void onFailure(String source, Exception e) { + logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to create snapshot-v2", repositoryName, snapshotName), e); + listener.onFailure(e); + if (enteredLoop) { + leaveRepoLoop(repositoryName); + } + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) { + final ShardGenerations shardGenerations = buildShardsGenerationFromRepositoryData( + newState.metadata(), + newState.routingTable(), + newEntry.indices(), + repositoryData + ); + final List dataStreams = indexNameExpressionResolver.dataStreamNames( + newState, + request.indicesOptions(), + request.indices() + ); final SnapshotInfo snapshotInfo = new SnapshotInfo( - snapshot.getSnapshotId(), + snapshotId, shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), - dataStreams, + newEntry.dataStreams(), pinnedTimestamp, null, System.currentTimeMillis(), shardGenerations.totalShards(), Collections.emptyList(), request.includeGlobalState(), - userMeta, + newEntry.userMetadata(), true, pinnedTimestamp ); - if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { - throw new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2, no longer cluster manager"); - } + final Version version = minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null); final StepListener pinnedTimestampListener = new StepListener<>(); - pinnedTimestampListener.whenComplete(repoData -> { listener.onResponse(snapshotInfo); }, listener::onFailure); - repository.finalizeSnapshot( - shardGenerations, - repositoryData.getGenId(), - metadataForSnapshot(currentState.metadata(), request.includeGlobalState(), false, dataStreams, indexIds), - snapshotInfo, - version, - state -> state, - Priority.IMMEDIATE, - new ActionListener() { - @Override - public void onResponse(RepositoryData repositoryData) { - if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { - failSnapshotCompletionListeners( - snapshot, - new SnapshotException(snapshot, "Aborting snapshot-v2, no longer cluster manager") - ); - listener.onFailure( - new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2, no longer cluster manager") - ); - return; + pinnedTimestampListener.whenComplete(repoData -> { + repository.finalizeSnapshot( + shardGenerations, + repositoryData.getGenId(), + metadataForSnapshot(newState.metadata(), request.includeGlobalState(), false, dataStreams, newEntry.indices()), + snapshotInfo, + version, + state -> stateWithoutSnapshot(state, snapshot), + Priority.IMMEDIATE, + new ActionListener() { + @Override + public void onResponse(RepositoryData repositoryData) { + leaveRepoLoop(repositoryName); + if (clusterService.state().nodes().isLocalNodeElectedClusterManager() == false) { + failSnapshotCompletionListeners( + snapshot, + new SnapshotException(snapshot, "Aborting snapshot-v2, no longer cluster manager") + ); + listener.onFailure( + new SnapshotException( + repositoryName, + snapshotName, + "Aborting snapshot-v2, no longer cluster manager" + ) + ); + return; + } + listener.onResponse(snapshotInfo); } - updateSnapshotPinnedTimestamp(repositoryData, snapshot, pinnedTimestamp, pinnedTimestampListener); - } - @Override - public void onFailure(Exception e) { - logger.error("Failed to upload files to snapshot repo {} for snapshot-v2 {} ", repositoryName, snapshotName); - listener.onFailure(e); + @Override + public void onFailure(Exception e) { + logger.error("Failed to finalize snapshot repo {} for snapshot-v2 {} ", repositoryName, snapshotName); + leaveRepoLoop(repositoryName); + // cleaning up in progress snapshot here + stateWithoutSnapshotV2(newState); + listener.onFailure(e); + } } - } - ); + ); + }, e -> { + logger.error("Failed to update pinned timestamp for snapshot-v2 {} {} {} ", repositoryName, snapshotName, e); + leaveRepoLoop(repositoryName); + // cleaning up in progress snapshot here + stateWithoutSnapshotV2(newState); + listener.onFailure(e); + }); + updateSnapshotPinnedTimestamp(repositoryData, snapshot, pinnedTimestamp, pinnedTimestampListener); + } - }, listener::onFailure); - } catch (Exception e) { - assert false : new AssertionError(e); - logger.error("Snapshot-v2 {} creation failed with exception {}", snapshot.getSnapshotId().getName(), e); - listener.onFailure(e); - } + @Override + public TimeValue timeout() { + return request.clusterManagerNodeTimeout(); + } + + }, "create_snapshot [" + snapshotName + ']', listener::onFailure); } private void createSnapshotPreValidations( @@ -770,12 +842,24 @@ public void cloneSnapshotV2( private SnapshotId sourceSnapshotId; private List indicesForSnapshot; + boolean enteredRepoLoop; + @Override public ClusterState execute(ClusterState currentState) { createSnapshotPreValidations(currentState, repositoryData, repositoryName, snapshotName); final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); final List runningSnapshots = snapshots.entries(); + // Entering finalize loop here to prevent concurrent snapshots v2 snapshots + enteredRepoLoop = tryEnterRepoLoop(repositoryName); + if (enteredRepoLoop == false) { + throw new ConcurrentSnapshotExecutionException( + repositoryName, + snapshotName, + "cannot start snapshot-v2 while a repository is in finalization state" + ); + } + sourceSnapshotId = repositoryData.getSnapshotIds() .stream() .filter(src -> src.getName().equals(request.source())) @@ -799,14 +883,14 @@ public ClusterState execute(ClusterState currentState) { indicesForSnapshot.add(indexId.getName()); } } - newEntry = SnapshotsInProgress.startClone( snapshot, sourceSnapshotId, repositoryData.resolveIndices(indicesForSnapshot), threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), - minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null) + minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null), + true ); final List newEntries = new ArrayList<>(runningSnapshots); newEntries.add(newEntry); @@ -817,6 +901,9 @@ public ClusterState execute(ClusterState currentState) { public void onFailure(String source, Exception e) { logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to clone snapshot-v2", repositoryName, snapshotName), e); listener.onFailure(e); + if (enteredRepoLoop) { + leaveRepoLoop(repositoryName); + } } @Override @@ -843,67 +930,80 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl true, snapshotInfo.getPinnedTimestamp() ); - if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { + if (clusterService.state().nodes().isLocalNodeElectedClusterManager() == false) { throw new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2 clone, no longer cluster manager"); } final StepListener pinnedTimestampListener = new StepListener<>(); pinnedTimestampListener.whenComplete(repoData -> { - logger.info("snapshot-v2 clone [{}] completed successfully", snapshot); - listener.onResponse(null); - }, listener::onFailure); - repository.finalizeSnapshot( - shardGenerations, - repositoryData.getGenId(), - metadataForSnapshot( - currentState.metadata(), - newEntry.includeGlobalState(), - false, - newEntry.dataStreams(), - newEntry.indices() - ), - cloneSnapshotInfo, - repositoryData.getVersion(sourceSnapshotId), - state -> stateWithoutSnapshot(state, snapshot), - Priority.IMMEDIATE, - new ActionListener() { - @Override - public void onResponse(RepositoryData repositoryData) { - if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { - failSnapshotCompletionListeners( - snapshot, - new SnapshotException(snapshot, "Aborting Snapshot-v2 clone, no longer cluster manager") - ); - listener.onFailure( - new SnapshotException( - repositoryName, - snapshotName, - "Aborting Snapshot-v2 clone, no longer cluster manager" - ) - ); - return; + repository.finalizeSnapshot( + shardGenerations, + repositoryData.getGenId(), + metadataForSnapshot( + currentState.metadata(), + newEntry.includeGlobalState(), + false, + newEntry.dataStreams(), + newEntry.indices() + ), + cloneSnapshotInfo, + repositoryData.getVersion(sourceSnapshotId), + state -> stateWithoutSnapshot(state, snapshot), + Priority.IMMEDIATE, + new ActionListener() { + @Override + public void onResponse(RepositoryData repositoryData) { + leaveRepoLoop(repositoryName); + if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { + failSnapshotCompletionListeners( + snapshot, + new SnapshotException(snapshot, "Aborting Snapshot-v2 clone, no longer cluster manager") + ); + listener.onFailure( + new SnapshotException( + repositoryName, + snapshotName, + "Aborting Snapshot-v2 clone, no longer cluster manager" + ) + ); + return; + } + logger.info("snapshot-v2 clone [{}] completed successfully", snapshot); + listener.onResponse(null); } - cloneSnapshotPinnedTimestamp( - repositoryData, - sourceSnapshotId, - snapshot, - snapshotInfo.getPinnedTimestamp(), - pinnedTimestampListener - ); - } - @Override - public void onFailure(Exception e) { - logger.error( - "Failed to upload files to snapshot repo {} for clone snapshot-v2 {} ", - repositoryName, - snapshotName - ); - listener.onFailure(e); + @Override + public void onFailure(Exception e) { + logger.error( + "Failed to upload files to snapshot repo {} for clone snapshot-v2 {} ", + repositoryName, + snapshotName + ); + stateWithoutSnapshotV2(newState); + leaveRepoLoop(repositoryName); + listener.onFailure(e); + } } - } + ); + }, e -> { + logger.error("Failed to update pinned timestamp for snapshot-v2 {} {} ", repositoryName, snapshotName); + stateWithoutSnapshotV2(newState); + leaveRepoLoop(repositoryName); + listener.onFailure(e); + }); + + cloneSnapshotPinnedTimestamp( + repositoryData, + sourceSnapshotId, + snapshot, + snapshotInfo.getPinnedTimestamp(), + pinnedTimestampListener ); - - }, listener::onFailure); + }, e -> { + logger.error("Failed to retrieve snapshot info for snapshot-v2 {} {} ", repositoryName, snapshotName); + stateWithoutSnapshotV2(newState); + leaveRepoLoop(repositoryName); + listener.onFailure(e); + }); } @Override @@ -1486,6 +1586,13 @@ public void applyClusterState(ClusterChangedEvent event) { // cluster-manager SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); final boolean newClusterManager = event.previousState().nodes().isLocalNodeElectedClusterManager() == false; + + if (newClusterManager && snapshotsInProgress.entries().isEmpty() == false) { + // clean up snapshot v2 in progress or clone v2 present. + // Snapshot v2 create and clone are sync operation . In case of cluster manager failures in midst , we won't + // send ack to caller and won't continue on new cluster manager . Caller will need to retry it. + stateWithoutSnapshotV2(event.state()); + } processExternalChanges( newClusterManager || removedNodesCleanupNeeded(snapshotsInProgress, event.nodesDelta().removedNodes()), event.routingTableChanged() && waitingShardsStartedOrUnassigned(snapshotsInProgress, event) @@ -1597,7 +1704,14 @@ private void processExternalChanges(boolean changedNodes, boolean startShards) { @Override public ClusterState execute(ClusterState currentState) { RoutingTable routingTable = currentState.routingTable(); - final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + // Removing shallow snapshots v2 as we we take care of these in stateWithoutSnapshotV2() + snapshots = SnapshotsInProgress.of( + snapshots.entries() + .stream() + .filter(snapshot -> snapshot.remoteStoreIndexShallowCopyV2() == false) + .collect(Collectors.toList()) + ); DiscoveryNodes nodes = currentState.nodes(); boolean changed = false; final EnumSet statesToUpdate; @@ -1654,7 +1768,7 @@ public ClusterState execute(ClusterState currentState) { changed = true; logger.debug("[{}] was found in dangling INIT or ABORTED state", snapshot); } else { - if (snapshot.state().completed() || completed(snapshot.shards().values())) { + if ((snapshot.state().completed() || completed(snapshot.shards().values()))) { finishedSnapshots.add(snapshot); } updatedSnapshotEntries.add(snapshot); @@ -2180,6 +2294,59 @@ private static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot sn return readyDeletions(result).v1(); } + private void stateWithoutSnapshotV2(ClusterState state) { + SnapshotsInProgress snapshots = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + boolean changed = false; + ArrayList entries = new ArrayList<>(); + for (SnapshotsInProgress.Entry entry : snapshots.entries()) { + if (entry.remoteStoreIndexShallowCopyV2()) { + changed = true; + } else { + entries.add(entry); + } + } + if (changed) { + logger.info("Cleaning up in progress v2 snapshots now"); + clusterService.submitStateUpdateTask( + "remove in progress snapshot v2 after cluster manager switch", + new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + SnapshotsInProgress snapshots = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + boolean changed = false; + ArrayList entries = new ArrayList<>(); + for (SnapshotsInProgress.Entry entry : snapshots.entries()) { + if (entry.remoteStoreIndexShallowCopyV2()) { + changed = true; + } else { + entries.add(entry); + } + } + if (changed) { + return ClusterState.builder(currentState) + .putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(unmodifiableList(entries))) + .build(); + } else { + return currentState; + } + } + + @Override + public void onFailure(String source, Exception e) { + // execute never fails , so we should never hit this. + logger.warn( + () -> new ParameterizedMessage( + "failed to remove in progress snapshot v2 state after cluster manager switch {}", + e + ), + e + ); + } + } + ); + } + } + /** * Removes record of running snapshot from cluster state and notifies the listener when this action is complete. This method is only * used when the snapshot fails for some reason. During normal operation the snapshot repository will remove the @@ -3330,6 +3497,9 @@ public boolean assertAllListenersResolved() { + " on [" + localNode + "]"; + if (repositoryOperations.isEmpty() == false) { + logger.info("Not empty"); + } assert repositoryOperations.isEmpty() : "Found leaked snapshots to finalize " + repositoryOperations + " on [" + localNode + "]"; return true; } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java index 8fd1f44286094..d79cb62b6b7ac 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -214,7 +214,11 @@ public void testSerDeRemoteStoreIndexShallowCopy() throws IOException { assert (curr_entry.remoteStoreIndexShallowCopy() == false); } } - try (StreamInput in = out.bytes().streamInput()) { + + BytesStreamOutput out2 = new BytesStreamOutput(); + out2.setVersion(Version.V_2_9_0); + snapshotsInProgress.writeTo(out2); + try (StreamInput in = out2.bytes().streamInput()) { in.setVersion(Version.V_2_9_0); actualSnapshotsInProgress = new SnapshotsInProgress(in); assert in.available() == 0; From 8d19cb9bc177d068e12fd7d5ac060da91c6bbe33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 08:53:05 -0400 Subject: [PATCH 065/108] Bump org.roaringbitmap:RoaringBitmap from 1.2.1 to 1.3.0 in /server (#16040) * Bump org.roaringbitmap:RoaringBitmap from 1.2.1 to 1.3.0 in /server Bumps [org.roaringbitmap:RoaringBitmap](https://github.com/RoaringBitmap/RoaringBitmap) from 1.2.1 to 1.3.0. - [Release notes](https://github.com/RoaringBitmap/RoaringBitmap/releases) - [Commits](https://github.com/RoaringBitmap/RoaringBitmap/compare/1.2.1...1.3.0) --- updated-dependencies: - dependency-name: org.roaringbitmap:RoaringBitmap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + server/build.gradle | 2 +- server/licenses/RoaringBitmap-1.2.1.jar.sha1 | 1 - server/licenses/RoaringBitmap-1.3.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 server/licenses/RoaringBitmap-1.2.1.jar.sha1 create mode 100644 server/licenses/RoaringBitmap-1.3.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index e88f85453109e..798802f11ebad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) +- Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040)) - Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038)) - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) diff --git a/server/build.gradle b/server/build.gradle index 0cc42ad690eab..83a04ef12d13b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -127,7 +127,7 @@ dependencies { api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}" // https://mvnrepository.com/artifact/org.roaringbitmap/RoaringBitmap - implementation 'org.roaringbitmap:RoaringBitmap:1.2.1' + implementation 'org.roaringbitmap:RoaringBitmap:1.3.0' testImplementation(project(":test:framework")) { // tests use the locally compiled version of server diff --git a/server/licenses/RoaringBitmap-1.2.1.jar.sha1 b/server/licenses/RoaringBitmap-1.2.1.jar.sha1 deleted file mode 100644 index ef8cd48c7a388..0000000000000 --- a/server/licenses/RoaringBitmap-1.2.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -828eb489b5e8c8762f2471010e9c7f20c7de596d \ No newline at end of file diff --git a/server/licenses/RoaringBitmap-1.3.0.jar.sha1 b/server/licenses/RoaringBitmap-1.3.0.jar.sha1 new file mode 100644 index 0000000000000..2e48289c4f7af --- /dev/null +++ b/server/licenses/RoaringBitmap-1.3.0.jar.sha1 @@ -0,0 +1 @@ +a46ce7a2dc494da69700ab421f081b1583857f6d \ No newline at end of file From 0363a64ac988a7bfb96a651e810d3b74cef65c2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 19:03:38 -0400 Subject: [PATCH 066/108] Bump com.nimbusds:nimbus-jose-jwt in /test/fixtures/hdfs-fixture (#16095) Bumps [com.nimbusds:nimbus-jose-jwt](https://bitbucket.org/connect2id/nimbus-jose-jwt) from 9.40 to 9.41.1. - [Changelog](https://bitbucket.org/connect2id/nimbus-jose-jwt/src/master/CHANGELOG.txt) - [Commits](https://bitbucket.org/connect2id/nimbus-jose-jwt/branches/compare/9.41.1..9.40) --- updated-dependencies: - dependency-name: com.nimbusds:nimbus-jose-jwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test/fixtures/hdfs-fixture/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 21fe28bfd835d..99b0386a48808 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -79,7 +79,7 @@ dependencies { api "org.jboss.xnio:xnio-nio:3.8.16.Final" api 'org.jline:jline:3.26.3' api 'org.apache.commons:commons-configuration2:2.11.0' - api 'com.nimbusds:nimbus-jose-jwt:9.40' + api 'com.nimbusds:nimbus-jose-jwt:9.41.1' api ('org.apache.kerby:kerb-admin:2.1.0') { exclude group: "org.jboss.xnio" exclude group: "org.jline" From 78860ace6cf7512c3e83a82bf116fc03ddb9e35d Mon Sep 17 00:00:00 2001 From: Sarat Vemulapalli Date: Fri, 27 Sep 2024 16:52:52 -0700 Subject: [PATCH 067/108] Registering disable user-agent setting with OpenSearch (#16052) * Registering allowlist setting in user-agent Signed-off-by: Sarat Vemulapalli * Adding unit tests Signed-off-by: Sarat Vemulapalli --------- Signed-off-by: Sarat Vemulapalli --- .../ingest/geoip/IngestGeoIpModulePluginTests.java | 7 +++++++ .../ingest/useragent/IngestUserAgentModulePlugin.java | 3 ++- .../ingest/useragent/IngestUserAgentModulePluginTests.java | 7 +++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java b/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java index 9446ec1228532..ac6b87aefb3ec 100644 --- a/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java +++ b/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java @@ -35,6 +35,7 @@ import com.maxmind.geoip2.model.AbstractResponse; import org.opensearch.common.network.InetAddresses; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.env.TestEnvironment; import org.opensearch.ingest.Processor; @@ -126,6 +127,12 @@ public void testAllowListNotSpecified() throws IOException { } } + public void testSettingsRegistration() { + final IngestGeoIpModulePlugin plugin = new IngestGeoIpModulePlugin(); + final List> settings = plugin.getSettings(); + assertTrue(settings.contains(IngestGeoIpModulePlugin.PROCESSORS_ALLOWLIST_SETTING)); + } + private void runAllowListTest(List allowList) throws IOException { Settings.Builder settingsBuilder = Settings.builder(); createDb(settingsBuilder); diff --git a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java index bac90d20b44e1..f5e2e34278880 100644 --- a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java @@ -44,6 +44,7 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.StandardOpenOption; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -152,6 +153,6 @@ static Map createUserAgentParsers(Path userAgentConfigD @Override public List> getSettings() { - return Collections.singletonList(CACHE_SIZE_SETTING); + return Arrays.asList(CACHE_SIZE_SETTING, PROCESSORS_ALLOWLIST_SETTING); } } diff --git a/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java b/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java index 31fdafff1188a..563158026c37f 100644 --- a/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java +++ b/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java @@ -8,6 +8,7 @@ package org.opensearch.ingest.useragent; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.env.TestEnvironment; import org.opensearch.ingest.Processor; @@ -89,6 +90,12 @@ public void testAllowListNotSpecified() throws IOException { } } + public void testSettingsRegistration() { + final IngestUserAgentModulePlugin plugin = new IngestUserAgentModulePlugin(); + final List> settings = plugin.getSettings(); + assertTrue(settings.contains(IngestUserAgentModulePlugin.PROCESSORS_ALLOWLIST_SETTING)); + } + private void runAllowListTest(List allowList) throws IOException { final Settings settings = settingsBuilder.putList(IngestUserAgentModulePlugin.PROCESSORS_ALLOWLIST_SETTING.getKey(), allowList) .build(); From b50117b0649a05792d5abc72da70991989726657 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Sep 2024 08:58:20 -0400 Subject: [PATCH 068/108] Bump com.maxmind.geoip2:geoip2 from 4.2.0 to 4.2.1 in /modules/ingest-geoip (#16042) * Bump com.maxmind.geoip2:geoip2 in /modules/ingest-geoip Bumps [com.maxmind.geoip2:geoip2](https://github.com/maxmind/GeoIP2-java) from 4.2.0 to 4.2.1. - [Release notes](https://github.com/maxmind/GeoIP2-java/releases) - [Changelog](https://github.com/maxmind/GeoIP2-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/maxmind/GeoIP2-java/compare/v4.2.0...v4.2.1) --- updated-dependencies: - dependency-name: com.maxmind.geoip2:geoip2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + modules/ingest-geoip/build.gradle | 2 +- modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 | 1 - modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 798802f11ebad..e35ca9c2fcb64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038)) - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) +- Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index c0ff155ce1038..162ac6f9ad620 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -39,7 +39,7 @@ opensearchplugin { } dependencies { - api('com.maxmind.geoip2:geoip2:4.2.0') + api('com.maxmind.geoip2:geoip2:4.2.1') // geoip2 dependencies: api('com.maxmind.db:maxmind-db:3.1.0') api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}") diff --git a/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 deleted file mode 100644 index b6bfeeb9da60b..0000000000000 --- a/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -78ff932dc13ac41dd1f0fd9e7405a7f4ad815ce0 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 new file mode 100644 index 0000000000000..a9dc5483ac727 --- /dev/null +++ b/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 @@ -0,0 +1 @@ +9dbf8a8bea88a33e88c46eb3f503721b4bd08b90 \ No newline at end of file From 1563e1af2d8ac2e4fdf555e4f81e389560cedc05 Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar <50844303+rahulkarajgikar@users.noreply.github.com> Date: Sat, 28 Sep 2024 18:44:35 +0530 Subject: [PATCH 069/108] Fix for race condition in node-join/node-left loop (#15521) * Add custom connect to node for handleJoinRequest Signed-off-by: Rahul Karajgikar --- CHANGELOG.md | 1 + .../cluster/coordination/NodeJoinLeftIT.java | 355 ++++++++++++++++++ .../cluster/NodeConnectionsService.java | 21 +- .../cluster/coordination/Coordinator.java | 23 +- .../cluster/coordination/Publication.java | 2 +- .../PublicationTransportHandler.java | 2 +- .../service/ClusterApplierService.java | 4 + .../org/opensearch/discovery/Discovery.java | 5 + .../main/java/org/opensearch/node/Node.java | 1 + .../transport/ClusterConnectionManager.java | 30 ++ .../transport/ConnectionManager.java | 4 + .../transport/RemoteConnectionManager.java | 10 + .../transport/TransportService.java | 12 + .../cluster/NodeConnectionsServiceTests.java | 191 ++++++++++ .../snapshots/SnapshotResiliencyTests.java | 28 +- .../org/opensearch/test/NoopDiscovery.java | 6 + .../ClusterConnectionManagerTests.java | 44 +++ .../AbstractCoordinatorTestCase.java | 32 +- .../coordination/DeterministicTaskQueue.java | 11 + .../org/opensearch/test/TestLogsAppender.java | 74 ++++ .../transport/StubbableConnectionManager.java | 10 + 21 files changed, 844 insertions(+), 22 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java create mode 100644 test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e35ca9c2fcb64..07637eaae3306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988)) - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) +- Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521)) ### Security diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java new file mode 100644 index 0000000000000..014e2bf642a4d --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java @@ -0,0 +1,355 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.cluster.coordination; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.cluster.NodeConnectionsService; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.MockEngineFactoryPlugin; +import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.plugins.Plugin; +import org.opensearch.tasks.Task; +import org.opensearch.test.InternalSettingsPlugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; +import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.TestLogsAppender; +import org.opensearch.test.store.MockFSIndexStore; +import org.opensearch.test.transport.MockTransportService; +import org.opensearch.test.transport.StubbableTransport; +import org.opensearch.transport.ClusterConnectionManager; +import org.opensearch.transport.TransportChannel; +import org.opensearch.transport.TransportRequest; +import org.opensearch.transport.TransportRequestHandler; +import org.opensearch.transport.TransportService; +import org.junit.After; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.opensearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_ACTION_NAME; +import static org.hamcrest.Matchers.is; + +/** + Check https://github.com/opensearch-project/OpenSearch/issues/4874 and + https://github.com/opensearch-project/OpenSearch/pull/15521 for context + */ +@ClusterScope(scope = Scope.TEST, numDataNodes = 0) +public class NodeJoinLeftIT extends OpenSearchIntegTestCase { + + private TestLogsAppender testLogsAppender; + private String clusterManager; + private String redNodeName; + private LoggerContext loggerContext; + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + MockTransportService.TestPlugin.class, + MockFSIndexStore.TestPlugin.class, + InternalSettingsPlugin.class, + MockEngineFactoryPlugin.class + ); + } + + @Override + protected void beforeIndexDeletion() throws Exception { + super.beforeIndexDeletion(); + internalCluster().assertConsistentHistoryBetweenTranslogAndLuceneIndex(); + internalCluster().assertSeqNos(); + internalCluster().assertSameDocIdsOnShards(); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + // Add any other specific messages you want to capture + List messagesToCapture = Arrays.asList("failed to join", "IllegalStateException"); + testLogsAppender = new TestLogsAppender(messagesToCapture); + loggerContext = (LoggerContext) LogManager.getContext(false); + Configuration config = loggerContext.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName()); + loggerConfig.addAppender(testLogsAppender, null, null); + loggerContext.updateLoggers(); + + String indexName = "test"; + final Settings nodeSettings = Settings.builder() + .put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(), "100ms") + .put(NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), "10s") + .put(FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), "200ms") + .put(FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), "100ms") + .put(FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), 1) + .put(NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), "100ms") + .build(); + // start a 3 node cluster with 1 cluster-manager + this.clusterManager = internalCluster().startNode(nodeSettings); + internalCluster().startNode(Settings.builder().put("node.attr.color", "blue").put(nodeSettings).build()); + this.redNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "red").put(nodeSettings).build()); + + // validate the 3 node cluster is up + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes(">=3").get(); + assertThat(response.isTimedOut(), is(false)); + + // create an index + client().admin() + .indices() + .prepareCreate(indexName) + .setSettings( + Settings.builder() + .put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + .get(); + } + + @After + public void tearDown() throws Exception { + testLogsAppender.clearCapturedLogs(); + loggerContext = (LoggerContext) LogManager.getContext(false); + Configuration config = loggerContext.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName()); + loggerConfig.removeAppender(testLogsAppender.getName()); + loggerContext.updateLoggers(); + super.tearDown(); + } + + public void testClusterStabilityWhenJoinRequestHappensDuringNodeLeftTask() throws Exception { + + ClusterService clusterManagerClsService = internalCluster().getInstance(ClusterService.class, clusterManager); + // Simulate a slow applier on the cm to delay node-left state application + clusterManagerClsService.addStateApplier(event -> { + if (event.nodesRemoved()) { + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }); + // Toggle to succeed/fail the followerchecker to simulate the initial node leaving. + AtomicBoolean succeedFollowerChecker = new AtomicBoolean(); + + // Simulate followerchecker failure on 1 node when succeedFollowerChecker is false + FollowerCheckerBehaviour simulatedFailureBehaviour = new FollowerCheckerBehaviour(() -> { + if (succeedFollowerChecker.get()) { + return; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new NodeHealthCheckFailureException("fake followerchecker failure simulated by test to repro race condition"); + }); + MockTransportService redTransportService = (MockTransportService) internalCluster().getInstance( + TransportService.class, + redNodeName + ); + redTransportService.addRequestHandlingBehavior(FOLLOWER_CHECK_ACTION_NAME, simulatedFailureBehaviour); + + // Loop runs 5 times to ensure race condition gets reproduced + testLogsAppender.clearCapturedLogs(); + for (int i = 0; i < 5; i++) { + logger.info("--> simulating followerchecker failure to trigger node-left"); + succeedFollowerChecker.set(false); + ClusterHealthResponse response1 = client().admin().cluster().prepareHealth().setWaitForNodes("2").get(); + assertThat(response1.isTimedOut(), is(false)); + + // once we know a node has left, we can re-enable followerchecker to work normally and validate node rejoins + logger.info("--> re-enabling normal followerchecker and validating cluster is stable"); + succeedFollowerChecker.set(true); + ClusterHealthResponse response2 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response2.isTimedOut(), is(false)); + + Thread.sleep(1000); + // checking again to validate stability and ensure node did not leave + ClusterHealthResponse response3 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response3.isTimedOut(), is(false)); + } + + succeedFollowerChecker.set(true); + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response.isTimedOut(), is(false)); + + // assert that join requests fail with the right exception + boolean logFound = testLogsAppender.waitForLog("failed to join", 30, TimeUnit.SECONDS) + && testLogsAppender.waitForLog( + "IllegalStateException[cannot make a new connection as disconnect to node", + 30, + TimeUnit.SECONDS + ); + assertTrue("Expected log was not found within the timeout period", logFound); + } + + public void testClusterStabilityWhenDisconnectDuringSlowNodeLeftTask() throws Exception { + ClusterService clusterManagerClsService = internalCluster().getInstance(ClusterService.class, clusterManager); + // Simulate a slow applier on the cm to delay node-left state application + clusterManagerClsService.addStateApplier(event -> { + if (event.nodesRemoved()) { + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }); + // Toggle to succeed/fail the followerchecker to simulate the initial node leaving. + AtomicBoolean succeedFollowerChecker = new AtomicBoolean(); + + // Simulate followerchecker failure on 1 node when succeedFollowerChecker is false + FollowerCheckerBehaviour simulatedFailureBehaviour = new FollowerCheckerBehaviour(() -> { + if (succeedFollowerChecker.get()) { + return; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new NodeHealthCheckFailureException("fake followerchecker failure simulated by test to repro race condition"); + }); + MockTransportService cmTransportService = (MockTransportService) internalCluster().getInstance( + TransportService.class, + clusterManager + ); + MockTransportService redTransportService = (MockTransportService) internalCluster().getInstance( + TransportService.class, + redNodeName + ); + redTransportService.addRequestHandlingBehavior(FOLLOWER_CHECK_ACTION_NAME, simulatedFailureBehaviour); + + // Loop runs 5 times to ensure race condition gets reproduced + testLogsAppender.clearCapturedLogs(); + for (int i = 0; i < 5; i++) { + // Fail followerchecker by force to trigger node disconnect and node left + logger.info("--> simulating followerchecker failure to trigger node-left"); + succeedFollowerChecker.set(false); + Thread.sleep(1000); + + // Trigger a node disconnect while node-left task is still processing + logger.info( + "--> triggering a simulated disconnect on red node, after the follower checker failed to see how node-left task deals with this" + ); + cmTransportService.disconnectFromNode(redTransportService.getLocalDiscoNode()); + + ClusterHealthResponse response1 = client().admin().cluster().prepareHealth().setWaitForNodes("2").get(); + assertThat(response1.isTimedOut(), is(false)); + + // once we know a node has left, we can re-enable followerchecker to work normally and validate node rejoins + logger.info("--> re-enabling normal followerchecker and validating cluster is stable"); + succeedFollowerChecker.set(true); + ClusterHealthResponse response2 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response2.isTimedOut(), is(false)); + + Thread.sleep(1000); + // checking again to validate stability and ensure node did not leave + ClusterHealthResponse response3 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response3.isTimedOut(), is(false)); + } + + succeedFollowerChecker.set(true); + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response.isTimedOut(), is(false)); + + // assert that join requests fail with the right exception + boolean logFound = testLogsAppender.waitForLog("failed to join", 30, TimeUnit.SECONDS); + assertTrue("Expected log was not found within the timeout period", logFound); + logFound = testLogsAppender.waitForLog( + "IllegalStateException[cannot make a new connection as disconnect to node", + 30, + TimeUnit.SECONDS + ); + assertTrue("Expected log was not found within the timeout period", logFound); + } + + public void testRestartDataNode() throws Exception { + + Settings redNodeDataPathSettings = internalCluster().dataPathSettings(redNodeName); + logger.info("-> stopping data node"); + internalCluster().stopRandomNode(settings -> settings.get("node.name").equals(redNodeName)); + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("2").get(); + assertThat(response.isTimedOut(), is(false)); + + logger.info("-> restarting stopped node"); + internalCluster().startNode(Settings.builder().put("node.name", redNodeName).put(redNodeDataPathSettings).build()); + response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response.isTimedOut(), is(false)); + } + + public void testRestartCmNode() throws Exception { + + Settings cmNodeSettings = internalCluster().dataPathSettings(clusterManager); + + logger.info("-> stopping cluster-manager node"); + internalCluster().stopRandomNode(settings -> settings.get("node.name").equals(clusterManager)); + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("2").get(); + assertThat(response.isTimedOut(), is(false)); + + logger.info("-> restarting stopped node"); + internalCluster().startNode(Settings.builder().put("node.name", clusterManager).put(cmNodeSettings).build()); + response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response.isTimedOut(), is(false)); + } + + private class FollowerCheckerBehaviour implements StubbableTransport.RequestHandlingBehavior { + private final Runnable connectionBreaker; + + private FollowerCheckerBehaviour(Runnable connectionBreaker) { + this.connectionBreaker = connectionBreaker; + } + + @Override + public void messageReceived( + TransportRequestHandler handler, + TransportRequest request, + TransportChannel channel, + Task task + ) throws Exception { + + connectionBreaker.run(); + handler.messageReceived(request, channel, task); + } + } +} diff --git a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java index 1c12c260b3929..8ce11c8183cf6 100644 --- a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java +++ b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java @@ -103,10 +103,10 @@ public class NodeConnectionsService extends AbstractLifecycleComponent { // contains an entry for every node in the latest cluster state, as well as for nodes from which we are in the process of // disconnecting - private final Map targetsByNode = new HashMap<>(); + protected final Map targetsByNode = new HashMap<>(); private final TimeValue reconnectInterval; - private volatile ConnectionChecker connectionChecker; + protected volatile ConnectionChecker connectionChecker; @Inject public NodeConnectionsService(Settings settings, ThreadPool threadPool, TransportService transportService) { @@ -115,6 +115,11 @@ public NodeConnectionsService(Settings settings, ThreadPool threadPool, Transpor this.reconnectInterval = NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(settings); } + // exposed for testing + protected ConnectionTarget createConnectionTarget(DiscoveryNode discoveryNode) { + return new ConnectionTarget(discoveryNode); + } + /** * Connect to all the given nodes, but do not disconnect from any extra nodes. Calls the completion handler on completion of all * connection attempts to _new_ nodes, but not on attempts to re-establish connections to nodes that are already known. @@ -159,6 +164,14 @@ public void connectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletion) runnables.forEach(Runnable::run); } + public void setPendingDisconnections(Set nodes) { + nodes.forEach(transportService::setPendingDisconnection); + } + + public void clearPendingDisconnections() { + transportService.clearPendingDisconnections(); + } + /** * Disconnect from any nodes to which we are currently connected which do not appear in the given nodes. Does not wait for the * disconnections to complete, because they might have to wait for ongoing connection attempts first. @@ -211,7 +224,7 @@ private void awaitPendingActivity(Runnable onCompletion) { * nodes which are in the process of disconnecting. The onCompletion handler is called after all ongoing connection/disconnection * attempts have completed. */ - private void connectDisconnectedTargets(Runnable onCompletion) { + protected void connectDisconnectedTargets(Runnable onCompletion) { final List runnables = new ArrayList<>(); synchronized (mutex) { final Collection connectionTargets = targetsByNode.values(); @@ -321,7 +334,7 @@ private enum ActivityType { * * @opensearch.internal */ - private class ConnectionTarget { + protected class ConnectionTarget { private final DiscoveryNode discoveryNode; private PlainListenableActionFuture future = PlainListenableActionFuture.newListenableFuture(); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 13a57d93f03f0..9859abe503eaa 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -42,6 +42,7 @@ import org.opensearch.cluster.ClusterStateTaskConfig; import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.LocalClusterUpdateTask; +import org.opensearch.cluster.NodeConnectionsService; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.coordination.ClusterFormationFailureHelper.ClusterFormationState; import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion; @@ -187,6 +188,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private final NodeHealthService nodeHealthService; private final PersistedStateRegistry persistedStateRegistry; private final RemoteStoreNodeService remoteStoreNodeService; + private NodeConnectionsService nodeConnectionsService; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -418,7 +420,11 @@ PublishWithJoinResponse handlePublishRequest(PublishRequest publishRequest) { synchronized (mutex) { final DiscoveryNode sourceNode = publishRequest.getAcceptedState().nodes().getClusterManagerNode(); - logger.trace("handlePublishRequest: handling [{}] from [{}]", publishRequest, sourceNode); + logger.debug( + "handlePublishRequest: handling version [{}] from [{}]", + publishRequest.getAcceptedState().getVersion(), + sourceNode + ); if (sourceNode.equals(getLocalNode()) && mode != Mode.LEADER) { // Rare case in which we stood down as leader between starting this publication and receiving it ourselves. The publication @@ -630,7 +636,6 @@ private void handleJoinRequest(JoinRequest joinRequest, JoinHelper.JoinCallback transportService.connectToNode(joinRequest.getSourceNode(), ActionListener.wrap(ignore -> { final ClusterState stateForJoinValidation = getStateForClusterManagerService(); - if (stateForJoinValidation.nodes().isLocalNodeElectedClusterManager()) { onJoinValidators.forEach(a -> a.accept(joinRequest.getSourceNode(), stateForJoinValidation)); if (stateForJoinValidation.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) == false) { @@ -814,6 +819,10 @@ public void onFailure(String source, Exception e) { public ClusterTasksResult execute(ClusterState currentState) { if (currentState.nodes().isLocalNodeElectedClusterManager() == false) { allocationService.cleanCaches(); + // This set only needs to be maintained on active cluster-manager + // This is cleaned up to avoid stale entries which would block future reconnections + logger.trace("Removing all pending disconnections as part of cluster-manager cleanup"); + nodeConnectionsService.clearPendingDisconnections(); } return unchanged(); } @@ -914,11 +923,18 @@ public DiscoveryStats stats() { @Override public void startInitialJoin() { synchronized (mutex) { + logger.trace("Starting initial join, becoming candidate"); becomeCandidate("startInitialJoin"); } clusterBootstrapService.scheduleUnconfiguredBootstrap(); } + @Override + public void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) { + assert this.nodeConnectionsService == null : "nodeConnectionsService is already set"; + this.nodeConnectionsService = nodeConnectionsService; + } + @Override protected void doStop() { configuredHostsResolver.stop(); @@ -1356,6 +1372,9 @@ assert getLocalNode().equals(clusterState.getNodes().get(getLocalNode().getId()) currentPublication = Optional.of(publication); final DiscoveryNodes publishNodes = publishRequest.getAcceptedState().nodes(); + // marking pending disconnects before publish + // if a nodes tries to send a joinRequest while it is pending disconnect, it should fail + nodeConnectionsService.setPendingDisconnections(new HashSet<>(clusterChangedEvent.nodesDelta().removedNodes())); leaderChecker.setCurrentNodes(publishNodes); followersChecker.setCurrentNodes(publishNodes); lagDetector.setTrackedNodes(publishNodes); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Publication.java b/server/src/main/java/org/opensearch/cluster/coordination/Publication.java index 43801a05dbc24..3f7218939be92 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Publication.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Publication.java @@ -85,7 +85,7 @@ public Publication(PublishRequest publishRequest, AckListener ackListener, LongS } public void start(Set faultyNodes) { - logger.trace("publishing {} to {}", publishRequest, publicationTargets); + logger.debug("publishing version {} to {}", publishRequest.getAcceptedState().getVersion(), publicationTargets); for (final DiscoveryNode faultyNode : faultyNodes) { onFaultyNode(faultyNode); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index cdf331b7bb577..caed2b6eceb49 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -542,7 +542,7 @@ public String executor() { } public void sendClusterState(DiscoveryNode destination, ActionListener listener) { - logger.debug("sending cluster state over transport to node: {}", destination.getName()); + logger.trace("sending cluster state over transport to node: {}", destination.getName()); if (sendFullVersion || previousState.nodes().nodeExists(destination) == false) { logger.trace("sending full cluster state version [{}] to [{}]", newState.version(), destination); sendFullClusterState(destination, listener); diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java index 47080cfbde692..d0b6f812e9ee2 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java @@ -502,6 +502,7 @@ private void runTask(UpdateTask task) { try { applyChanges(task, previousClusterState, newClusterState, stopWatch); TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, currentTimeInMillis() - startTimeMS)); + // At this point, cluster state appliers and listeners are completed logger.debug( "processing [{}]: took [{}] done applying updated cluster state (version: {}, uuid: {})", task.source, @@ -510,6 +511,7 @@ private void runTask(UpdateTask task) { newClusterState.stateUUID() ); warnAboutSlowTaskIfNeeded(executionTime, task.source, stopWatch); + // Then we call the ClusterApplyListener of the task task.listener.onSuccess(task.source); } catch (Exception e) { TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, currentTimeInMillis() - startTimeMS)); @@ -578,6 +580,7 @@ private void applyChanges(UpdateTask task, ClusterState previousClusterState, Cl logger.debug("apply cluster state with version {}", newClusterState.version()); callClusterStateAppliers(clusterChangedEvent, stopWatch); + logger.debug("completed calling appliers of cluster state for version {}", newClusterState.version()); nodeConnectionsService.disconnectFromNodesExcept(newClusterState.nodes()); @@ -594,6 +597,7 @@ private void applyChanges(UpdateTask task, ClusterState previousClusterState, Cl state.set(newClusterState); callClusterStateListeners(clusterChangedEvent, stopWatch); + logger.debug("completed calling listeners of cluster state for version {}", newClusterState.version()); } protected void connectToNodesAndWait(ClusterState newClusterState) { diff --git a/server/src/main/java/org/opensearch/discovery/Discovery.java b/server/src/main/java/org/opensearch/discovery/Discovery.java index 9d6807b6522c9..6d9fb1f4985df 100644 --- a/server/src/main/java/org/opensearch/discovery/Discovery.java +++ b/server/src/main/java/org/opensearch/discovery/Discovery.java @@ -32,6 +32,7 @@ package org.opensearch.discovery; +import org.opensearch.cluster.NodeConnectionsService; import org.opensearch.cluster.coordination.ClusterStatePublisher; import org.opensearch.common.lifecycle.LifecycleComponent; @@ -54,4 +55,8 @@ public interface Discovery extends LifecycleComponent, ClusterStatePublisher { */ void startInitialJoin(); + /** + * Sets the NodeConnectionsService which is an abstraction used for connection management + */ + void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index a8d4ebcf23dab..4962d72d8728a 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -1602,6 +1602,7 @@ public Node start() throws NodeValidationException { injector.getInstance(GatewayService.class).start(); Discovery discovery = injector.getInstance(Discovery.class); + discovery.setNodeConnectionsService(nodeConnectionsService); clusterService.getClusterManagerService().setClusterStatePublisher(discovery::publish); // Start the transport service now so the publish address will be added to the local disco node in ClusterService diff --git a/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java b/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java index e634323d58269..3a3e8c964b6c5 100644 --- a/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java +++ b/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java @@ -64,6 +64,15 @@ public class ClusterConnectionManager implements ConnectionManager { private final ConcurrentMap connectedNodes = ConcurrentCollections.newConcurrentMap(); private final ConcurrentMap> pendingConnections = ConcurrentCollections.newConcurrentMap(); + /** + This set is used only by cluster-manager nodes. + Nodes are marked as pending disconnect right before cluster state publish phase. + They are cleared up as part of cluster state apply commit phase + This is to avoid connections from being made to nodes that are in the process of leaving the cluster + Note: If a disconnect is initiated while a connect is in progress, this Set will not handle this case. + Callers need to ensure that connects and disconnects are sequenced. + */ + private final Set pendingDisconnections = ConcurrentCollections.newConcurrentSet(); private final AbstractRefCounted connectingRefCounter = new AbstractRefCounted("connection manager") { @Override protected void closeInternal() { @@ -122,12 +131,19 @@ public void connectToNode( ConnectionValidator connectionValidator, ActionListener listener ) throws ConnectTransportException { + logger.trace("connecting to node [{}]", node); ConnectionProfile resolvedProfile = ConnectionProfile.resolveConnectionProfile(connectionProfile, defaultProfile); if (node == null) { listener.onFailure(new ConnectTransportException(null, "can't connect to a null node")); return; } + // if node-left is still in progress, we fail the connect request early + if (pendingDisconnections.contains(node)) { + listener.onFailure(new IllegalStateException("cannot make a new connection as disconnect to node [" + node + "] is pending")); + return; + } + if (connectingRefCounter.tryIncRef() == false) { listener.onFailure(new IllegalStateException("connection manager is closed")); return; @@ -170,6 +186,7 @@ public void connectToNode( conn.addCloseListener(ActionListener.wrap(() -> { logger.trace("unregistering {} after connection close and marking as disconnected", node); connectedNodes.remove(node, finalConnection); + pendingDisconnections.remove(node); connectionListener.onNodeDisconnected(node, conn); })); } @@ -226,6 +243,19 @@ public void disconnectFromNode(DiscoveryNode node) { // if we found it and removed it we close nodeChannels.close(); } + pendingDisconnections.remove(node); + logger.trace("Removed node [{}] from pending disconnections list", node); + } + + @Override + public void setPendingDisconnection(DiscoveryNode node) { + logger.trace("marking disconnection as pending for node: [{}]", node); + pendingDisconnections.add(node); + } + + @Override + public void clearPendingDisconnections() { + pendingDisconnections.clear(); } /** diff --git a/server/src/main/java/org/opensearch/transport/ConnectionManager.java b/server/src/main/java/org/opensearch/transport/ConnectionManager.java index 10cfc2907098f..ebd5ccf29c8cc 100644 --- a/server/src/main/java/org/opensearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/opensearch/transport/ConnectionManager.java @@ -65,6 +65,10 @@ void connectToNode( void disconnectFromNode(DiscoveryNode node); + void setPendingDisconnection(DiscoveryNode node); + + void clearPendingDisconnections(); + Set getAllConnectedNodes(); int size(); diff --git a/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java b/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java index bd646f10df517..52f29bea8050d 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java +++ b/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java @@ -114,6 +114,16 @@ public void disconnectFromNode(DiscoveryNode node) { delegate.disconnectFromNode(node); } + @Override + public void setPendingDisconnection(DiscoveryNode node) { + delegate.setPendingDisconnection(node); + } + + @Override + public void clearPendingDisconnections() { + delegate.clearPendingDisconnections(); + } + @Override public ConnectionProfile getConnectionProfile() { return delegate.getConnectionProfile(); diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index fff6d82b23c7e..fe8631aa5ca3d 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -773,6 +773,18 @@ public void disconnectFromNode(DiscoveryNode node) { connectionManager.disconnectFromNode(node); } + public void setPendingDisconnection(DiscoveryNode node) { + connectionManager.setPendingDisconnection(node); + } + + /** + * Wipes out all pending disconnections. + * This is called on cluster-manager failover to remove stale entries + */ + public void clearPendingDisconnections() { + connectionManager.clearPendingDisconnections(); + } + public void addMessageListener(TransportMessageListener listener) { messageListener.listeners.add(listener); } diff --git a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java index 4cf82f1dabab3..4500860c937ea 100644 --- a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java @@ -35,6 +35,9 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; import org.opensearch.OpenSearchTimeoutException; import org.opensearch.Version; import org.opensearch.action.support.PlainActionFuture; @@ -53,9 +56,11 @@ import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.TestLogsAppender; import org.opensearch.test.junit.annotations.TestLogging; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.ClusterConnectionManager; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.ConnectionProfile; import org.opensearch.transport.Transport; @@ -69,6 +74,7 @@ import org.junit.Before; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -77,6 +83,7 @@ import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import static java.util.Collections.emptySet; @@ -86,12 +93,15 @@ import static org.opensearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; import static org.opensearch.node.Node.NODE_NAME_SETTING; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; public class NodeConnectionsServiceTests extends OpenSearchTestCase { private ThreadPool threadPool; private TransportService transportService; private Map> nodeConnectionBlocks; + private TestLogsAppender testLogsAppender; + private LoggerContext loggerContext; private List generateNodes() { List nodes = new ArrayList<>(); @@ -490,6 +500,108 @@ public void testDebugLogging() throws IllegalAccessException { } } + public void testConnectionCheckerRetriesIfPendingDisconnection() throws InterruptedException { + final Settings.Builder settings = Settings.builder(); + final long reconnectIntervalMillis = 50; + settings.put(CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), reconnectIntervalMillis + "ms"); + + final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue( + builder().put(NODE_NAME_SETTING.getKey(), "node").build(), + random() + ); + + MockTransport transport = new MockTransport(deterministicTaskQueue.getThreadPool()); + TestTransportService transportService = new TestTransportService(transport, deterministicTaskQueue.getThreadPool()); + transportService.start(); + transportService.acceptIncomingRequests(); + + final TestNodeConnectionsService service = new TestNodeConnectionsService( + settings.build(), + deterministicTaskQueue.getThreadPool(), + transportService + ); + service.start(); + + // setup the connections + final DiscoveryNode node = new DiscoveryNode("node0", buildNewFakeTransportAddress(), Version.CURRENT); + + final DiscoveryNodes nodes = DiscoveryNodes.builder().add(node).build(); + + final AtomicBoolean connectionCompleted = new AtomicBoolean(); + service.connectToNodes(nodes, () -> connectionCompleted.set(true)); + deterministicTaskQueue.runAllRunnableTasks(); + assertTrue(connectionCompleted.get()); + + // reset any logs as we want to assert for exceptions that show up after this + // reset connect to node count to assert for later + logger.info("--> resetting captured logs and counters"); + testLogsAppender.clearCapturedLogs(); + // this ensures we only track connection attempts that happen after the disconnection + transportService.resetConnectToNodeCallCount(); + + // block connection checker reconnection attempts until after we set pending disconnections + logger.info("--> disabling connection checker, and triggering disconnect"); + service.setShouldReconnect(false); + transportService.disconnectFromNode(node); + + // set pending disconnections to true to fail future reconnection attempts + final long maxDisconnectionTime = 1000; + deterministicTaskQueue.scheduleNow(new Runnable() { + @Override + public void run() { + logger.info("--> setting pending disconnections to fail next connection attempts"); + service.setPendingDisconnections(new HashSet<>(Collections.singleton(node))); + } + + @Override + public String toString() { + return "scheduled disconnection of " + node; + } + }); + // our task queue will have the first task as the runnable to set pending disconnections + // here we re-enable the connection checker to enqueue next tasks for attempting reconnection + logger.info("--> re-enabling reconnection checker"); + service.setShouldReconnect(true); + + final long maxReconnectionTime = 2000; + final int expectedReconnectionAttempts = 10; + + // this will first run the task to set the pending disconnections, then will execute the reconnection tasks + // exit early when we have enough reconnection attempts + logger.info("--> running tasks in order until expected reconnection attempts"); + runTasksInOrderUntilExpectedReconnectionAttempts( + deterministicTaskQueue, + maxDisconnectionTime + maxReconnectionTime, + transportService, + expectedReconnectionAttempts + ); + logger.info("--> verifying that connectionchecker tried to reconnect"); + + // assert that the connections failed + assertFalse("connected to " + node, transportService.nodeConnected(node)); + + // assert that we saw at least the required number of reconnection attempts, and the exceptions that showed up are as expected + logger.info("--> number of reconnection attempts: {}", transportService.getConnectToNodeCallCount()); + assertThat( + "Did not see enough reconnection attempts from connection checker", + transportService.getConnectToNodeCallCount(), + greaterThan(expectedReconnectionAttempts) + ); + boolean logFound = testLogsAppender.waitForLog("failed to connect", 1, TimeUnit.SECONDS) + && testLogsAppender.waitForLog( + "IllegalStateException: cannot make a new connection as disconnect to node", + 1, + TimeUnit.SECONDS + ); + assertTrue("Expected log for reconnection failure was not found in the required time period", logFound); + + // clear the pending disconnections and ensure the connection gets re-established automatically by connectionchecker + logger.info("--> clearing pending disconnections to allow connections to re-establish"); + service.clearPendingDisconnections(); + runTasksUntil(deterministicTaskQueue, maxDisconnectionTime + maxReconnectionTime + 2 * reconnectIntervalMillis); + assertConnectedExactlyToNodes(transportService, nodes); + } + private void runTasksUntil(DeterministicTaskQueue deterministicTaskQueue, long endTimeMillis) { while (deterministicTaskQueue.getCurrentTimeMillis() < endTimeMillis) { if (deterministicTaskQueue.hasRunnableTasks() && randomBoolean()) { @@ -501,6 +613,24 @@ private void runTasksUntil(DeterministicTaskQueue deterministicTaskQueue, long e deterministicTaskQueue.runAllRunnableTasks(); } + private void runTasksInOrderUntilExpectedReconnectionAttempts( + DeterministicTaskQueue deterministicTaskQueue, + long endTimeMillis, + TestTransportService transportService, + int expectedReconnectionAttempts + ) { + // break the loop if we timeout or if we have enough reconnection attempts + while ((deterministicTaskQueue.getCurrentTimeMillis() < endTimeMillis) + && (transportService.getConnectToNodeCallCount() <= expectedReconnectionAttempts)) { + if (deterministicTaskQueue.hasRunnableTasks() && randomBoolean()) { + deterministicTaskQueue.runNextTask(); + } else if (deterministicTaskQueue.hasDeferredTasks()) { + deterministicTaskQueue.advanceTime(); + } + } + deterministicTaskQueue.runAllRunnableTasksInEnqueuedOrder(); + } + private void ensureConnections(NodeConnectionsService service) { final PlainActionFuture future = new PlainActionFuture<>(); service.ensureConnections(() -> future.onResponse(null)); @@ -526,6 +656,16 @@ private void assertConnected(TransportService transportService, Iterable messagesToCapture = Arrays.asList("failed to connect", "IllegalStateException"); + testLogsAppender = new TestLogsAppender(messagesToCapture); + loggerContext = (LoggerContext) LogManager.getContext(false); + Configuration config = loggerContext.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(NodeConnectionsService.class.getName()); + loggerConfig.addAppender(testLogsAppender, null, null); + loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName()); + loggerConfig.addAppender(testLogsAppender, null, null); + loggerContext.updateLoggers(); ThreadPool threadPool = new TestThreadPool(getClass().getName()); this.threadPool = threadPool; nodeConnectionBlocks = newConcurrentMap(); @@ -537,6 +677,14 @@ public void setUp() throws Exception { @Override @After public void tearDown() throws Exception { + testLogsAppender.clearCapturedLogs(); + loggerContext = (LoggerContext) LogManager.getContext(false); + Configuration config = loggerContext.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(NodeConnectionsService.class.getName()); + loggerConfig.removeAppender(testLogsAppender.getName()); + loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName()); + loggerConfig.removeAppender(testLogsAppender.getName()); + loggerContext.updateLoggers(); transportService.stop(); ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); threadPool = null; @@ -545,6 +693,8 @@ public void tearDown() throws Exception { private final class TestTransportService extends TransportService { + private final AtomicInteger connectToNodeCallCount = new AtomicInteger(0); + private TestTransportService(Transport transport, ThreadPool threadPool) { super( Settings.EMPTY, @@ -588,6 +738,47 @@ public void connectToNode(DiscoveryNode node, ActionListener listener) thr } else { super.connectToNode(node, listener); } + logger.info("calling connectToNode"); + connectToNodeCallCount.incrementAndGet(); + } + + public int getConnectToNodeCallCount() { + return connectToNodeCallCount.get(); + } + + public void resetConnectToNodeCallCount() { + connectToNodeCallCount.set(0); + } + } + + private class TestNodeConnectionsService extends NodeConnectionsService { + private boolean shouldReconnect = true; + + public TestNodeConnectionsService(Settings settings, ThreadPool threadPool, TransportService transportService) { + super(settings, threadPool, transportService); + } + + public void setShouldReconnect(boolean shouldReconnect) { + this.shouldReconnect = shouldReconnect; + } + + @Override + protected void doStart() { + final StoppableConnectionChecker connectionChecker = new StoppableConnectionChecker(); + this.connectionChecker = connectionChecker; + connectionChecker.scheduleNextCheck(); + } + + class StoppableConnectionChecker extends NodeConnectionsService.ConnectionChecker { + @Override + protected void doRun() { + if (connectionChecker == this && shouldReconnect) { + connectDisconnectedTargets(this::scheduleNextCheck); + } else { + // Skip reconnection attempt but still schedule the next check + scheduleNextCheck(); + } + } } } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 350c6f9ae8f6b..440227436175d 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -1923,11 +1923,6 @@ private final class TestClusterNode { protected PrioritizedOpenSearchThreadPoolExecutor createThreadPoolExecutor() { return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue, threadPool); } - - @Override - protected void connectToNodesAndWait(ClusterState newClusterState) { - // don't do anything, and don't block - } } ); recoverySettings = new RecoverySettings(settings, clusterSettings); @@ -2094,7 +2089,7 @@ public void onFailure(final Exception e) { rerouteService, threadPool ); - nodeConnectionsService = new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService); + nodeConnectionsService = createTestNodeConnectionsService(clusterService.getSettings(), threadPool, transportService); final MetadataMappingService metadataMappingService = new MetadataMappingService(clusterService, indicesService); indicesClusterStateService = new IndicesClusterStateService( settings, @@ -2492,6 +2487,24 @@ protected void assertSnapshotOrGenericThread() { } } + public NodeConnectionsService createTestNodeConnectionsService( + Settings settings, + ThreadPool threadPool, + TransportService transportService + ) { + return new NodeConnectionsService(settings, threadPool, transportService) { + @Override + public void connectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletion) { + // just update targetsByNode to ensure disconnect runs for these nodes + // we rely on disconnect to run for keeping track of pendingDisconnects and ensuring node-joins can happen + for (final DiscoveryNode discoveryNode : discoveryNodes) { + this.targetsByNode.put(discoveryNode, createConnectionTarget(discoveryNode)); + } + onCompletion.run(); + } + }; + } + public ClusterInfoService getMockClusterInfoService() { return clusterInfoService; } @@ -2563,10 +2576,11 @@ public void start(ClusterState initialState) { new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), null ); + coordinator.setNodeConnectionsService(nodeConnectionsService); clusterManagerService.setClusterStatePublisher(coordinator); - coordinator.start(); clusterService.getClusterApplierService().setNodeConnectionsService(nodeConnectionsService); nodeConnectionsService.start(); + coordinator.start(); clusterService.start(); indicesService.start(); indicesClusterStateService.start(); diff --git a/server/src/test/java/org/opensearch/test/NoopDiscovery.java b/server/src/test/java/org/opensearch/test/NoopDiscovery.java index 42d3f1887ab4d..c35503a556db6 100644 --- a/server/src/test/java/org/opensearch/test/NoopDiscovery.java +++ b/server/src/test/java/org/opensearch/test/NoopDiscovery.java @@ -32,6 +32,7 @@ package org.opensearch.test; import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.NodeConnectionsService; import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.common.lifecycle.LifecycleListener; import org.opensearch.core.action.ActionListener; @@ -55,6 +56,11 @@ public void startInitialJoin() { } + @Override + public void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) { + + } + @Override public Lifecycle.State lifecycleState() { return null; diff --git a/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java b/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java index 1d734a56ef189..fdf762aa096f0 100644 --- a/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java +++ b/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java @@ -320,6 +320,50 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti assertEquals(0, nodeDisconnectedCount.get()); } + public void testConnectFailsWhenDisconnectIsPending() { + AtomicInteger nodeConnectedCount = new AtomicInteger(); + AtomicInteger nodeDisconnectedCount = new AtomicInteger(); + connectionManager.addListener(new TransportConnectionListener() { + @Override + public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) { + nodeConnectedCount.incrementAndGet(); + } + + @Override + public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + nodeDisconnectedCount.incrementAndGet(); + } + }); + + DiscoveryNode node = new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT); + ConnectionManager.ConnectionValidator validator = (c, p, l) -> l.onResponse(null); + Transport.Connection connection = new TestConnect(node); + doAnswer(invocationOnMock -> { + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(connection); + return null; + }).when(transport).openConnection(eq(node), eq(connectionProfile), any(ActionListener.class)); + assertFalse(connectionManager.nodeConnected(node)); + + // Mark connection as pending disconnect, any connection attempt should fail + connectionManager.setPendingDisconnection(node); + PlainActionFuture fut = new PlainActionFuture<>(); + connectionManager.connectToNode(node, connectionProfile, validator, fut); + expectThrows(IllegalStateException.class, () -> fut.actionGet()); + + // clear the pending disconnect and assert that connection succeeds + connectionManager.clearPendingDisconnections(); + assertFalse(connectionManager.nodeConnected(node)); + PlainActionFuture.get( + future -> connectionManager.connectToNode(node, connectionProfile, validator, ActionListener.map(future, x -> null)) + ); + assertFalse(connection.isClosed()); + assertTrue(connectionManager.nodeConnected(node)); + assertEquals(1, connectionManager.size()); + assertEquals(1, nodeConnectedCount.get()); + assertEquals(0, nodeDisconnectedCount.get()); + } + private static class TestConnect extends CloseableConnection { private final DiscoveryNode node; diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index b432e5411404e..3efcc538a1b25 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -55,6 +55,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterService; @@ -1150,9 +1151,12 @@ protected Optional getDisruptableMockTransport(Transpo new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) ); clusterService = new ClusterService(settings, clusterSettings, clusterManagerService, clusterApplierService); - clusterService.setNodeConnectionsService( - new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService) + NodeConnectionsService nodeConnectionsService = createTestNodeConnectionsService( + clusterService.getSettings(), + threadPool, + transportService ); + clusterService.setNodeConnectionsService(nodeConnectionsService); repositoriesService = new RepositoriesService( settings, clusterService, @@ -1187,6 +1191,7 @@ protected Optional getDisruptableMockTransport(Transpo new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), null ); + coordinator.setNodeConnectionsService(nodeConnectionsService); clusterManagerService.setClusterStatePublisher(coordinator); final GatewayService gatewayService = new GatewayService( settings, @@ -1588,6 +1593,24 @@ public void onNodeAck(DiscoveryNode node, Exception e) { } } + public static NodeConnectionsService createTestNodeConnectionsService( + Settings settings, + ThreadPool threadPool, + TransportService transportService + ) { + return new NodeConnectionsService(settings, threadPool, transportService) { + @Override + public void connectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletion) { + // just update targetsByNode to ensure disconnect runs for these nodes + // we rely on disconnect to run for keeping track of pendingDisconnects and ensuring node-joins can happen + for (final DiscoveryNode discoveryNode : discoveryNodes) { + this.targetsByNode.put(discoveryNode, createConnectionTarget(discoveryNode)); + } + onCompletion.run(); + } + }; + } + static class DisruptableClusterApplierService extends ClusterApplierService { private final String nodeName; private final DeterministicTaskQueue deterministicTaskQueue; @@ -1641,11 +1664,6 @@ public void onNewClusterState(String source, Supplier clusterState } } - @Override - protected void connectToNodesAndWait(ClusterState newClusterState) { - // don't do anything, and don't block - } - @Override protected boolean applicationMayFail() { return this.applicationMayFail; diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java index 1ad18bf89d5ba..4f692c7bc8f62 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java @@ -92,6 +92,12 @@ public void runAllRunnableTasks() { } } + public void runAllRunnableTasksInEnqueuedOrder() { + while (hasRunnableTasks()) { + runTask(0); + } + } + public void runAllTasks() { while (hasDeferredTasks() || hasRunnableTasks()) { if (hasDeferredTasks() && random.nextBoolean()) { @@ -141,6 +147,11 @@ public void runRandomTask() { runTask(RandomNumbers.randomIntBetween(random, 0, runnableTasks.size() - 1)); } + public void runNextTask() { + assert hasRunnableTasks(); + runTask(0); + } + private void runTask(final int index) { final Runnable task = runnableTasks.remove(index); logger.trace("running task {} of {}: {}", index, runnableTasks.size() + 1, task); diff --git a/test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java b/test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java new file mode 100644 index 0000000000000..030f399a5bcc0 --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Property; +import org.apache.logging.log4j.core.layout.PatternLayout; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Test logs appender that provides functionality to extract specific logs/exception messages and wait for it to show up + * @opensearch.internal + */ +public class TestLogsAppender extends AbstractAppender { + private final List capturedLogs = new ArrayList<>(); + private final List messagesToCapture; + + public TestLogsAppender(List messagesToCapture) { + super("TestAppender", null, PatternLayout.createDefaultLayout(), false, Property.EMPTY_ARRAY); + this.messagesToCapture = messagesToCapture; + start(); + } + + @Override + public void append(LogEvent event) { + if (shouldCaptureMessage(event.getMessage().getFormattedMessage())) capturedLogs.add(event.getMessage().getFormattedMessage()); + if (event.getThrown() != null) { + if (shouldCaptureMessage(event.getThrown().toString())) capturedLogs.add(event.getThrown().toString()); + for (StackTraceElement element : event.getThrown().getStackTrace()) + if (shouldCaptureMessage(element.toString())) capturedLogs.add(element.toString()); + } + } + + public boolean shouldCaptureMessage(String log) { + return messagesToCapture.stream().anyMatch(log::contains); + } + + public List getCapturedLogs() { + return new ArrayList<>(capturedLogs); + } + + public boolean waitForLog(String expectedLog, long timeout, TimeUnit unit) { + long startTime = System.currentTimeMillis(); + long timeoutInMillis = unit.toMillis(timeout); + + while (System.currentTimeMillis() - startTime < timeoutInMillis) { + if (capturedLogs.stream().anyMatch(log -> log.contains(expectedLog))) { + return true; + } + try { + Thread.sleep(100); // Wait for 100ms before checking again + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + return false; + } + + // Clear captured logs + public void clearCapturedLogs() { + capturedLogs.clear(); + } +} diff --git a/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java b/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java index 37df90fb103a3..d1e1a3e8af17c 100644 --- a/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java +++ b/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java @@ -123,6 +123,16 @@ public void disconnectFromNode(DiscoveryNode node) { delegate.disconnectFromNode(node); } + @Override + public void setPendingDisconnection(DiscoveryNode node) { + delegate.setPendingDisconnection(node); + } + + @Override + public void clearPendingDisconnections() { + delegate.clearPendingDisconnections(); + } + @Override public int size() { return delegate.size(); From d510b129ff66e0d77b4e7390c10bd309d14c9c1a Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Sat, 28 Sep 2024 22:36:37 -0400 Subject: [PATCH 070/108] Update Gradle to 8.10.2 (#16107) Signed-off-by: Andriy Redko --- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 5c6205ebf24d4..e312a2da77d94 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15 +distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6 From 031d8ae022e47aedc8563bb7735aa582c8145bde Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Mon, 30 Sep 2024 02:38:15 +0800 Subject: [PATCH 071/108] Fix testHAProxyModeConnectionWorks is flaky (#16087) * Fix testHAProxyModeConnectionWorks is flaky Signed-off-by: Gao Binlong * Optimize health check method Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- qa/remote-clusters/docker-compose.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qa/remote-clusters/docker-compose.yml b/qa/remote-clusters/docker-compose.yml index cf6aefcf5c1a3..2112da17efe6e 100644 --- a/qa/remote-clusters/docker-compose.yml +++ b/qa/remote-clusters/docker-compose.yml @@ -72,3 +72,11 @@ services: - "9600" volumes: - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + - opensearch-2 + healthcheck: + test: ["CMD", "timeout", "1", "bash", "-c", "cat < /dev/null > /dev/tcp/localhost/9600"] + interval: 2s + timeout: 1s + retries: 5 + start_period: 15s From 0dba562f896a7294fc16eddf64081fe45b3aa3bb Mon Sep 17 00:00:00 2001 From: gargharsh3134 <51459091+gargharsh3134@users.noreply.github.com> Date: Mon, 30 Sep 2024 08:34:27 +0530 Subject: [PATCH 072/108] Implementing pagination for _cat/indices API (#14718) * Implementing pagination for _cat/indices Signed-off-by: Harsh Garg --- CHANGELOG.md | 1 + .../org/opensearch/action/ActionModule.java | 14 +- .../java/org/opensearch/common/Table.java | 15 + .../java/org/opensearch/rest/RestHandler.java | 7 + .../java/org/opensearch/rest/RestRequest.java | 8 + .../rest/action/cat/RestIndicesAction.java | 163 +++++-- .../opensearch/rest/action/cat/RestTable.java | 41 +- .../rest/action/list/AbstractListAction.java | 77 ++++ .../action/list/RestIndicesListAction.java | 104 +++++ .../rest/action/list/RestListAction.java | 58 +++ .../rest/action/list/package-info.java | 12 + .../pagination/IndexPaginationStrategy.java | 185 ++++++++ .../rest/pagination/PageParams.java | 48 +++ .../opensearch/rest/pagination/PageToken.java | 42 ++ .../rest/pagination/PaginationStrategy.java | 75 ++++ .../rest/pagination/package-info.java | 12 + .../action/cat/RestIndicesActionTests.java | 70 ++- .../rest/action/cat/RestTableTests.java | 93 +++- .../IndexPaginationStrategyTests.java | 399 ++++++++++++++++++ 19 files changed, 1355 insertions(+), 69 deletions(-) create mode 100644 server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java create mode 100644 server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java create mode 100644 server/src/main/java/org/opensearch/rest/action/list/RestListAction.java create mode 100644 server/src/main/java/org/opensearch/rest/action/list/package-info.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/PageParams.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/PageToken.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/package-info.java create mode 100644 server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 07637eaae3306..b628e9277959d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) +- Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 3fe0f1dc7cb83..937d7509fe68c 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -461,6 +461,9 @@ import org.opensearch.rest.action.ingest.RestGetPipelineAction; import org.opensearch.rest.action.ingest.RestPutPipelineAction; import org.opensearch.rest.action.ingest.RestSimulatePipelineAction; +import org.opensearch.rest.action.list.AbstractListAction; +import org.opensearch.rest.action.list.RestIndicesListAction; +import org.opensearch.rest.action.list.RestListAction; import org.opensearch.rest.action.search.RestClearScrollAction; import org.opensearch.rest.action.search.RestCountAction; import org.opensearch.rest.action.search.RestCreatePitAction; @@ -802,9 +805,14 @@ private ActionFilters setupActionFilters(List actionPlugins) { public void initRestHandlers(Supplier nodesInCluster) { List catActions = new ArrayList<>(); + List listActions = new ArrayList<>(); Consumer registerHandler = handler -> { if (handler instanceof AbstractCatAction) { - catActions.add((AbstractCatAction) handler); + if (handler instanceof AbstractListAction && ((AbstractListAction) handler).isActionPaginated()) { + listActions.add((AbstractListAction) handler); + } else { + catActions.add((AbstractCatAction) handler); + } } restController.registerHandler(handler); }; @@ -980,6 +988,9 @@ public void initRestHandlers(Supplier nodesInCluster) { } registerHandler.accept(new RestTemplatesAction()); + // LIST API + registerHandler.accept(new RestIndicesListAction()); + // Point in time API registerHandler.accept(new RestCreatePitAction()); registerHandler.accept(new RestDeletePitAction()); @@ -1011,6 +1022,7 @@ public void initRestHandlers(Supplier nodesInCluster) { } } registerHandler.accept(new RestCatAction(catActions)); + registerHandler.accept(new RestListAction(listActions)); registerHandler.accept(new RestDecommissionAction()); registerHandler.accept(new RestGetDecommissionStateAction()); registerHandler.accept(new RestRemoteStoreStatsAction()); diff --git a/server/src/main/java/org/opensearch/common/Table.java b/server/src/main/java/org/opensearch/common/Table.java index da14f628efa0f..133ec3052e6c9 100644 --- a/server/src/main/java/org/opensearch/common/Table.java +++ b/server/src/main/java/org/opensearch/common/Table.java @@ -34,6 +34,7 @@ import org.opensearch.common.time.DateFormatter; import org.opensearch.core.common.Strings; +import org.opensearch.rest.pagination.PageToken; import java.time.Instant; import java.time.ZoneOffset; @@ -59,9 +60,19 @@ public class Table { private List currentCells; private boolean inHeaders = false; private boolean withTime = false; + /** + * paginatedQueryResponse if null will imply the Table response is not paginated. + */ + private PageToken pageToken; public static final String EPOCH = "epoch"; public static final String TIMESTAMP = "timestamp"; + public Table() {} + + public Table(@Nullable PageToken pageToken) { + this.pageToken = pageToken; + } + public Table startHeaders() { inHeaders = true; currentCells = new ArrayList<>(); @@ -230,6 +241,10 @@ public Map getAliasMap() { return headerAliasMap; } + public PageToken getPageToken() { + return pageToken; + } + /** * Cell in a table * diff --git a/server/src/main/java/org/opensearch/rest/RestHandler.java b/server/src/main/java/org/opensearch/rest/RestHandler.java index 1139e5fc65f31..143cbd472ed07 100644 --- a/server/src/main/java/org/opensearch/rest/RestHandler.java +++ b/server/src/main/java/org/opensearch/rest/RestHandler.java @@ -125,6 +125,13 @@ default boolean allowSystemIndexAccessByDefault() { return false; } + /** + * Denotes whether the RestHandler will output paginated responses or not. + */ + default boolean isActionPaginated() { + return false; + } + static RestHandler wrapper(RestHandler delegate) { return new Wrapper(delegate); } diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java index 2c397f7fc6e8e..f241b567c3204 100644 --- a/server/src/main/java/org/opensearch/rest/RestRequest.java +++ b/server/src/main/java/org/opensearch/rest/RestRequest.java @@ -51,6 +51,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.http.HttpChannel; import org.opensearch.http.HttpRequest; +import org.opensearch.rest.pagination.PageParams; import java.io.IOException; import java.io.InputStream; @@ -67,6 +68,9 @@ import static org.opensearch.common.unit.TimeValue.parseTimeValue; import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; +import static org.opensearch.rest.pagination.PageParams.PARAM_NEXT_TOKEN; +import static org.opensearch.rest.pagination.PageParams.PARAM_SIZE; +import static org.opensearch.rest.pagination.PageParams.PARAM_SORT; /** * REST Request @@ -591,6 +595,10 @@ public static MediaType parseContentType(List header) { throw new IllegalArgumentException("empty Content-Type header"); } + public PageParams parsePaginatedQueryParams(String defaultSortOrder, int defaultPageSize) { + return new PageParams(param(PARAM_NEXT_TOKEN), param(PARAM_SORT, defaultSortOrder), paramAsInt(PARAM_SIZE, defaultPageSize)); + } + /** * Thrown if there is an error in the content type header. * diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 9dc711f804144..1e76008ff8c64 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -50,6 +50,7 @@ import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Table; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; @@ -61,6 +62,9 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.rest.action.list.AbstractListAction; +import org.opensearch.rest.pagination.IndexPaginationStrategy; +import org.opensearch.rest.pagination.PageToken; import java.time.Instant; import java.time.ZoneOffset; @@ -68,9 +72,11 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.Spliterators; import java.util.function.Function; @@ -87,7 +93,7 @@ * * @opensearch.api */ -public class RestIndicesAction extends AbstractCatAction { +public class RestIndicesAction extends AbstractListAction { private static final DateFormatter STRICT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_time"); private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestIndicesAction.class); @@ -151,48 +157,70 @@ public RestResponse buildResponse(final Table table) throws Exception { new ActionListener() { @Override public void onResponse(final GetSettingsResponse getSettingsResponse) { - final GroupedActionListener groupedListener = createGroupedListener(request, 4, listener); - groupedListener.onResponse(getSettingsResponse); - // The list of indices that will be returned is determined by the indices returned from the Get Settings call. // All the other requests just provide additional detail, and wildcards may be resolved differently depending on the // type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so // force the IndicesOptions for all the sub-requests to be as inclusive as possible. final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden(); - // Indices that were successfully resolved during the get settings request might be deleted when the subsequent - // cluster - // state, cluster health and indices stats requests execute. We have to distinguish two cases: - // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the - // subsequent requests to fail. - // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests - // not to - // fail on the deleted index (as we want to ignore wildcards that cannot be resolved). - // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve - // the - // index names with the same indices options that we used for the initial cluster state request (strictExpand). - sendIndicesStatsRequest( - indices, - subRequestIndicesOptions, - includeUnloadedSegments, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); + // Indices that were successfully resolved during the get settings request might be deleted when the + // subsequent cluster state, cluster health and indices stats requests execute. We have to distinguish two cases: + // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we + // want the subsequent requests to fail. + // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent + // requests not to fail on the deleted index (as we want to ignore wildcards that cannot be resolved). + // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests + // re-resolve the index names with the same indices options that we used for the initial cluster state + // request (strictExpand). sendClusterStateRequest( indices, subRequestIndicesOptions, local, clusterManagerNodeTimeout, client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); - sendClusterHealthRequest( - indices, - subRequestIndicesOptions, - local, - clusterManagerNodeTimeout, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + new ActionListener() { + @Override + public void onResponse(ClusterStateResponse clusterStateResponse) { + IndexPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); + // For non-paginated queries, indicesToBeQueried would be same as indices retrieved from + // rest request and unresolved, while for paginated queries, it would be a list of indices + // already resolved by ClusterStateRequest and to be displayed in a page. + final String[] indicesToBeQueried = Objects.isNull(paginationStrategy) + ? indices + : paginationStrategy.getRequestedEntities().toArray(new String[0]); + final GroupedActionListener groupedListener = createGroupedListener( + request, + 4, + listener, + indicesToBeQueried, + Objects.isNull(paginationStrategy) ? null : paginationStrategy.getResponseToken() + ); + groupedListener.onResponse(getSettingsResponse); + groupedListener.onResponse(clusterStateResponse); + + sendIndicesStatsRequest( + indicesToBeQueried, + subRequestIndicesOptions, + includeUnloadedSegments, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + + sendClusterHealthRequest( + indicesToBeQueried, + subRequestIndicesOptions, + local, + clusterManagerNodeTimeout, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + } ); } @@ -203,6 +231,7 @@ public void onFailure(final Exception e) { } ); }; + } /** @@ -287,7 +316,9 @@ private void sendIndicesStatsRequest( private GroupedActionListener createGroupedListener( final RestRequest request, final int size, - final ActionListener listener + final ActionListener
listener, + final String[] indicesToBeQueried, + final PageToken pageToken ) { return new GroupedActionListener<>(new ActionListener>() { @Override @@ -311,7 +342,15 @@ public void onResponse(final Collection responses) { IndicesStatsResponse statsResponse = extractResponse(responses, IndicesStatsResponse.class); Map indicesStats = statsResponse.getIndices(); - Table responseTable = buildTable(request, indicesSettings, indicesHealths, indicesStats, indicesStates); + Table responseTable = buildTable( + request, + indicesSettings, + indicesHealths, + indicesStats, + indicesStates, + getTableIterator(indicesToBeQueried, indicesSettings), + pageToken + ); listener.onResponse(responseTable); } catch (Exception e) { onFailure(e); @@ -340,7 +379,11 @@ protected Set responseParams() { @Override protected Table getTableWithHeader(final RestRequest request) { - Table table = new Table(); + return getTableWithHeader(request, null); + } + + protected Table getTableWithHeader(final RestRequest request, final PageToken pageToken) { + Table table = new Table(pageToken); table.startHeaders(); table.addCell("health", "alias:h;desc:current health status"); table.addCell("status", "alias:s;desc:open/close status"); @@ -704,22 +747,27 @@ protected Table getTableWithHeader(final RestRequest request) { } // package private for testing - Table buildTable( + protected Table buildTable( final RestRequest request, final Map indicesSettings, final Map indicesHealths, final Map indicesStats, - final Map indicesMetadatas + final Map indicesMetadatas, + final Iterator> tableIterator, + final PageToken pageToken ) { - final String healthParam = request.param("health"); - final Table table = getTableWithHeader(request); + final Table table = getTableWithHeader(request, pageToken); + + while (tableIterator.hasNext()) { + final Tuple tuple = tableIterator.next(); + String indexName = tuple.v1(); + Settings settings = tuple.v2(); - indicesSettings.forEach((indexName, settings) -> { if (indicesMetadatas.containsKey(indexName) == false) { // the index exists in the Get Indices response but is not present in the cluster state: // it is likely that the index was deleted in the meanwhile, so we ignore it. - return; + continue; } final IndexMetadata indexMetadata = indicesMetadatas.get(indexName); @@ -748,7 +796,7 @@ Table buildTable( skip = ClusterHealthStatus.RED != healthStatusFilter; } if (skip) { - return; + continue; } } @@ -982,7 +1030,8 @@ Table buildTable( table.addCell(searchThrottled); table.endRow(); - }); + + } return table; } @@ -991,4 +1040,34 @@ Table buildTable( private static A extractResponse(final Collection responses, Class c) { return (A) responses.stream().filter(c::isInstance).findFirst().get(); } + + @Override + public boolean isActionPaginated() { + return false; + } + + protected IndexPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { + return null; + } + + /** + * Provides the iterator to be used for building the response table. + */ + protected Iterator> getTableIterator(String[] indices, Map indexSettingsMap) { + return new Iterator<>() { + final Iterator settingsMapIter = indexSettingsMap.keySet().iterator(); + + @Override + public boolean hasNext() { + return settingsMapIter.hasNext(); + } + + @Override + public Tuple next() { + String index = settingsMapIter.next(); + return new Tuple<>(index, indexSettingsMap.get(index)); + } + }; + } + } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java index 4f1090b163ee6..d622dd7a956f4 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java @@ -58,8 +58,11 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; +import static org.opensearch.rest.pagination.PageToken.PAGINATED_RESPONSE_NEXT_TOKEN_KEY; + /** * a REST table * @@ -87,8 +90,37 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel RestRequest request = channel.request(); XContentBuilder builder = channel.newBuilder(); List displayHeaders = buildDisplayHeaders(table, request); + if (Objects.nonNull(table.getPageToken())) { + buildPaginatedXContentBuilder(table, request, builder, displayHeaders); + } else { + builder.startArray(); + addRowsToXContentBuilder(table, request, builder, displayHeaders); + builder.endArray(); + } + return new BytesRestResponse(RestStatus.OK, builder); + } + + private static void buildPaginatedXContentBuilder( + Table table, + RestRequest request, + XContentBuilder builder, + List displayHeaders + ) throws Exception { + assert Objects.nonNull(table.getPageToken().getPaginatedEntity()) : "Paginated element is required in-case of paginated responses"; + builder.startObject(); + builder.field(PAGINATED_RESPONSE_NEXT_TOKEN_KEY, table.getPageToken().getNextToken()); + builder.startArray(table.getPageToken().getPaginatedEntity()); + addRowsToXContentBuilder(table, request, builder, displayHeaders); + builder.endArray(); + builder.endObject(); + } - builder.startArray(); + private static void addRowsToXContentBuilder( + Table table, + RestRequest request, + XContentBuilder builder, + List displayHeaders + ) throws Exception { List rowOrder = getRowOrder(table, request); for (Integer row : rowOrder) { builder.startObject(); @@ -97,8 +129,6 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel } builder.endObject(); } - builder.endArray(); - return new BytesRestResponse(RestStatus.OK, builder); } public static RestResponse buildTextPlainResponse(Table table, RestChannel channel) throws IOException { @@ -136,6 +166,11 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann } out.append("\n"); } + // Adding a new row for next_token, in the response if the table is paginated. + if (Objects.nonNull(table.getPageToken())) { + out.append("next_token" + " " + table.getPageToken().getNextToken()); + out.append("\n"); + } out.close(); return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOut.bytes()); } diff --git a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java new file mode 100644 index 0000000000000..f3d6d6653a550 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.cat.AbstractCatAction; +import org.opensearch.rest.pagination.PageParams; + +import java.io.IOException; +import java.util.Objects; + +import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; + +/** + * Base Transport action class for _list API. + * Serves as a base class for APIs wanting to support pagination. + * Existing _cat APIs can refer {@link org.opensearch.rest.action.cat.RestIndicesAction}. + * @opensearch.api + */ +public abstract class AbstractListAction extends AbstractCatAction { + + private static final int DEFAULT_PAGE_SIZE = 100; + protected PageParams pageParams; + + protected abstract void documentation(StringBuilder sb); + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + boolean helpWanted = request.paramAsBoolean("help", false); + if (helpWanted || isActionPaginated() == false) { + return super.prepareRequest(request, client); + } + this.pageParams = validateAndGetPageParams(request); + assert Objects.nonNull(pageParams) : "pageParams can not be null for paginated queries"; + return doCatRequest(request, client); + } + + @Override + public boolean isActionPaginated() { + return true; + } + + /** + * + * @return Metadata that can be extracted out from the rest request. Query params supported by the action specific + * to pagination along with any respective validations to be added here. + */ + protected PageParams validateAndGetPageParams(RestRequest restRequest) { + PageParams pageParams = restRequest.parsePaginatedQueryParams(defaultSort(), defaultPageSize()); + // validating pageSize + if (pageParams.getSize() <= 0) { + throw new IllegalArgumentException("size must be greater than zero"); + } + // Validating sort order + if (!(PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) || PARAM_DESC_SORT_VALUE.equals(pageParams.getSort()))) { + throw new IllegalArgumentException("value of sort can either be asc or desc"); + } + return pageParams; + } + + protected int defaultPageSize() { + return DEFAULT_PAGE_SIZE; + } + + protected String defaultSort() { + return PARAM_ASC_SORT_VALUE; + } + +} diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java new file mode 100644 index 0000000000000..ad5c58c86ce90 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -0,0 +1,104 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.settings.Settings; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.cat.RestIndicesAction; +import org.opensearch.rest.pagination.IndexPaginationStrategy; +import org.opensearch.rest.pagination.PageParams; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; +import static org.opensearch.rest.RestRequest.Method.GET; + +/** + * _list API action to output indices in pages. + * + * @opensearch.api + */ +public class RestIndicesListAction extends RestIndicesAction { + + private static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE = 5000; + private static final int DEFAULT_LIST_INDICES_PAGE_SIZE = 500; + + @Override + public List routes() { + return unmodifiableList(asList(new Route(GET, "/_list/indices"), new Route(GET, "/_list/indices/{index}"))); + } + + @Override + public String getName() { + return "list_indices_action"; + } + + @Override + protected void documentation(StringBuilder sb) { + sb.append("/_list/indices\n"); + sb.append("/_list/indices/{index}\n"); + } + + @Override + public boolean isActionPaginated() { + return true; + } + + @Override + protected PageParams validateAndGetPageParams(RestRequest restRequest) { + PageParams pageParams = super.validateAndGetPageParams(restRequest); + // validate max supported pageSize + if (pageParams.getSize() > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE) { + throw new IllegalArgumentException("size should be less than [" + MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE + "]"); + } + // Next Token in the request will be validated by the IndexStrategyToken itself. + if (Objects.nonNull(pageParams.getRequestedToken())) { + IndexPaginationStrategy.IndexStrategyToken.validateIndexStrategyToken(pageParams.getRequestedToken()); + } + return pageParams; + } + + protected int defaultPageSize() { + return DEFAULT_LIST_INDICES_PAGE_SIZE; + } + + @Override + protected IndexPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { + return new IndexPaginationStrategy(pageParams, clusterStateResponse.getState()); + } + + // Public for testing + @Override + public Iterator> getTableIterator(String[] indices, Map indexSettingsMap) { + return new Iterator<>() { + int indexPos = 0; + + @Override + public boolean hasNext() { + while (indexPos < indices.length && indexSettingsMap.containsKey(indices[indexPos]) == false) { + indexPos++; + } + return indexPos < indices.length; + } + + @Override + public Tuple next() { + Tuple tuple = new Tuple<>(indices[indexPos], indexSettingsMap.get(indices[indexPos])); + indexPos++; + return tuple; + } + }; + } +} diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java new file mode 100644 index 0000000000000..4b8551ea7e14a --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestRequest; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +/** + * Base _list API endpoint + * + * @opensearch.api + */ +public class RestListAction extends BaseRestHandler { + + private static final String LIST = ":‑|"; + private static final String LIST_NL = LIST + "\n"; + private final String HELP; + + public RestListAction(List listActions) { + StringBuilder sb = new StringBuilder(); + sb.append(LIST_NL); + for (AbstractListAction listAction : listActions) { + listAction.documentation(sb); + } + HELP = sb.toString(); + } + + @Override + public List routes() { + return singletonList(new Route(GET, "/_list")); + } + + @Override + public String getName() { + return "list_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, HELP)); + } + +} diff --git a/server/src/main/java/org/opensearch/rest/action/list/package-info.java b/server/src/main/java/org/opensearch/rest/action/list/package-info.java new file mode 100644 index 0000000000000..8d6563ff9b344 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * {@link org.opensearch.rest.RestHandler}s for actions that list out results in chunks of pages. + */ +package org.opensearch.rest.action.list; diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java new file mode 100644 index 0000000000000..f89ab14e4b24d --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java @@ -0,0 +1,185 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.Nullable; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; + +/** + * This strategy can be used by the Rest APIs wanting to paginate the responses based on Indices. + * The strategy considers create timestamps of indices as the keys to iterate over pages. + * + * @opensearch.internal + */ +public class IndexPaginationStrategy implements PaginationStrategy { + private static final String DEFAULT_INDICES_PAGINATED_ENTITY = "indices"; + + private static final Comparator ASC_COMPARATOR = (metadata1, metadata2) -> { + if (metadata1.getCreationDate() == metadata2.getCreationDate()) { + return metadata1.getIndex().getName().compareTo(metadata2.getIndex().getName()); + } + return Long.compare(metadata1.getCreationDate(), metadata2.getCreationDate()); + }; + private static final Comparator DESC_COMPARATOR = (metadata1, metadata2) -> { + if (metadata1.getCreationDate() == metadata2.getCreationDate()) { + return metadata2.getIndex().getName().compareTo(metadata1.getIndex().getName()); + } + return Long.compare(metadata2.getCreationDate(), metadata1.getCreationDate()); + }; + + private final PageToken pageToken; + private final List requestedIndices; + + public IndexPaginationStrategy(PageParams pageParams, ClusterState clusterState) { + // Get list of indices metadata sorted by their creation time and filtered by the last sent index + List sortedIndices = PaginationStrategy.getSortedIndexMetadata( + clusterState, + getMetadataFilter(pageParams.getRequestedToken(), pageParams.getSort()), + PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) ? ASC_COMPARATOR : DESC_COMPARATOR + ); + // Trim sortedIndicesList to get the list of indices metadata to be sent as response + List metadataSublist = getMetadataSubList(sortedIndices, pageParams.getSize()); + // Get list of index names from the trimmed metadataSublist + this.requestedIndices = metadataSublist.stream().map(metadata -> metadata.getIndex().getName()).collect(Collectors.toList()); + this.pageToken = getResponseToken( + pageParams.getSize(), + sortedIndices.size(), + metadataSublist.isEmpty() ? null : metadataSublist.get(metadataSublist.size() - 1) + ); + } + + private static Predicate getMetadataFilter(String requestedTokenStr, String sortOrder) { + boolean isAscendingSort = sortOrder.equals(PARAM_ASC_SORT_VALUE); + IndexStrategyToken requestedToken = Objects.isNull(requestedTokenStr) || requestedTokenStr.isEmpty() + ? null + : new IndexStrategyToken(requestedTokenStr); + if (Objects.isNull(requestedToken)) { + return indexMetadata -> true; + } + return metadata -> { + if (metadata.getIndex().getName().equals(requestedToken.lastIndexName)) { + return false; + } else if (metadata.getCreationDate() == requestedToken.lastIndexCreationTime) { + return isAscendingSort + ? metadata.getIndex().getName().compareTo(requestedToken.lastIndexName) > 0 + : metadata.getIndex().getName().compareTo(requestedToken.lastIndexName) < 0; + } + return isAscendingSort + ? metadata.getCreationDate() > requestedToken.lastIndexCreationTime + : metadata.getCreationDate() < requestedToken.lastIndexCreationTime; + }; + } + + private List getMetadataSubList(List sortedIndices, final int pageSize) { + if (sortedIndices.isEmpty()) { + return new ArrayList<>(); + } + return sortedIndices.subList(0, Math.min(pageSize, sortedIndices.size())); + } + + private PageToken getResponseToken(final int pageSize, final int totalIndices, IndexMetadata lastIndex) { + if (totalIndices <= pageSize) { + return new PageToken(null, DEFAULT_INDICES_PAGINATED_ENTITY); + } + return new PageToken( + new IndexStrategyToken(lastIndex.getCreationDate(), lastIndex.getIndex().getName()).generateEncryptedToken(), + DEFAULT_INDICES_PAGINATED_ENTITY + ); + } + + @Override + @Nullable + public PageToken getResponseToken() { + return pageToken; + } + + @Override + public List getRequestedEntities() { + return Objects.isNull(requestedIndices) ? new ArrayList<>() : requestedIndices; + } + + /** + * TokenParser to be used by {@link IndexPaginationStrategy}. + * Token would look like: CreationTimeOfLastRespondedIndex + | + NameOfLastRespondedIndex + */ + public static class IndexStrategyToken { + + private static final String JOIN_DELIMITER = "|"; + private static final String SPLIT_REGEX = "\\|"; + private static final int CREATE_TIME_POS_IN_TOKEN = 0; + private static final int INDEX_NAME_POS_IN_TOKEN = 1; + + /** + * Represents creation times of last index which was displayed in the page. + * Used to identify the new start point in case the indices get created/deleted while queries are executed. + */ + private final long lastIndexCreationTime; + + /** + * Represents name of the last index which was displayed in the page. + * Used to identify whether the sorted list of indices has changed or not. + */ + private final String lastIndexName; + + public IndexStrategyToken(String requestedTokenString) { + // TODO: Avoid validating the requested token multiple times while calling from Rest and/or Transport layer. + validateIndexStrategyToken(requestedTokenString); + String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); + final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX); + this.lastIndexCreationTime = Long.parseLong(decryptedTokenElements[CREATE_TIME_POS_IN_TOKEN]); + this.lastIndexName = decryptedTokenElements[INDEX_NAME_POS_IN_TOKEN]; + } + + public IndexStrategyToken(long creationTimeOfLastRespondedIndex, String nameOfLastRespondedIndex) { + Objects.requireNonNull(nameOfLastRespondedIndex, "index name should be provided"); + this.lastIndexCreationTime = creationTimeOfLastRespondedIndex; + this.lastIndexName = nameOfLastRespondedIndex; + } + + public String generateEncryptedToken() { + return PaginationStrategy.encryptStringToken(String.join(JOIN_DELIMITER, String.valueOf(lastIndexCreationTime), lastIndexName)); + } + + /** + * Will perform simple validations on token received in the request. + * Token should be base64 encoded, and should contain the expected number of elements separated by "|". + * Timestamps should also be a valid long. + * + * @param requestedTokenStr string denoting the encoded token requested by the user. + */ + public static void validateIndexStrategyToken(String requestedTokenStr) { + Objects.requireNonNull(requestedTokenStr, "requestedTokenString can not be null"); + String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenStr); + final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX); + if (decryptedTokenElements.length != 2) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + try { + long creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[CREATE_TIME_POS_IN_TOKEN]); + if (creationTimeOfLastRespondedIndex <= 0) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } catch (NumberFormatException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } + } + +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageParams.java b/server/src/main/java/org/opensearch/rest/pagination/PageParams.java new file mode 100644 index 0000000000000..9b2074bc3fed0 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/PageParams.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.common.annotation.PublicApi; + +/** + * + * Class specific to paginated queries, which will contain common query params required by a paginated API. + */ +@PublicApi(since = "3.0.0") +public class PageParams { + + public static final String PARAM_SORT = "sort"; + public static final String PARAM_NEXT_TOKEN = "next_token"; + public static final String PARAM_SIZE = "size"; + public static final String PARAM_ASC_SORT_VALUE = "asc"; + public static final String PARAM_DESC_SORT_VALUE = "desc"; + + private final String requestedTokenStr; + private final String sort; + private final int size; + + public PageParams(String requestedToken, String sort, int size) { + this.requestedTokenStr = requestedToken; + this.sort = sort; + this.size = size; + } + + public String getSort() { + return sort; + } + + public String getRequestedToken() { + return requestedTokenStr; + } + + public int getSize() { + return size; + } + +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageToken.java b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java new file mode 100644 index 0000000000000..d62e1be695715 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +/** + * Pagination response metadata for a paginated query. + * @opensearch.internal + */ +public class PageToken { + + public static final String PAGINATED_RESPONSE_NEXT_TOKEN_KEY = "next_token"; + + /** + * String denoting the next_token of paginated response, which will be used to fetch next page (if any). + */ + private final String nextToken; + + /** + * String denoting the element which is being paginated (for e.g. shards, indices..). + */ + private final String paginatedEntity; + + public PageToken(String nextToken, String paginatedElement) { + assert paginatedElement != null : "paginatedElement must be specified for a paginated response"; + this.nextToken = nextToken; + this.paginatedEntity = paginatedElement; + } + + public String getNextToken() { + return nextToken; + } + + public String getPaginatedEntity() { + return paginatedEntity; + } +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java new file mode 100644 index 0000000000000..7f9825a7cc09b --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; + +import java.util.Base64; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Interface to be implemented by any strategy getting used for paginating rest responses. + * + * @opensearch.internal + */ +public interface PaginationStrategy { + + String INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE = + "Parameter [next_token] has been tainted and is incorrect. Please provide a valid [next_token]."; + + /** + * + * @return Base64 encoded string, which can be used to fetch next page of response. + */ + PageToken getResponseToken(); + + /** + * + * @return List of elements fetched corresponding to the store and token received by the strategy. + */ + List getRequestedEntities(); + + /** + * + * Utility method to get list of indices filtered as per {@param filterPredicate} and the sorted according to {@param comparator}. + */ + static List getSortedIndexMetadata( + final ClusterState clusterState, + Predicate filterPredicate, + Comparator comparator + ) { + return clusterState.metadata().indices().values().stream().filter(filterPredicate).sorted(comparator).collect(Collectors.toList()); + } + + static String encryptStringToken(String tokenString) { + if (Objects.isNull(tokenString)) { + return null; + } + return Base64.getEncoder().encodeToString(tokenString.getBytes(UTF_8)); + } + + static String decryptStringToken(String encTokenString) { + if (Objects.isNull(encTokenString)) { + return null; + } + try { + return new String(Base64.getDecoder().decode(encTokenString), UTF_8); + } catch (IllegalArgumentException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/package-info.java b/server/src/main/java/org/opensearch/rest/pagination/package-info.java new file mode 100644 index 0000000000000..324b8a6c46f88 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Exposes utilities for Rest actions to paginate responses. + */ +package org.opensearch.rest.pagination; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 96b1c75371697..1d1b509ae94e5 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -47,9 +47,13 @@ import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; +import org.opensearch.rest.action.list.RestIndicesListAction; +import org.opensearch.rest.pagination.PageToken; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; +import org.junit.Before; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -63,13 +67,14 @@ public class RestIndicesActionTests extends OpenSearchTestCase { - public void testBuildTable() { - final int numIndices = randomIntBetween(3, 20); - final Map indicesSettings = new LinkedHashMap<>(); - final Map indicesMetadatas = new LinkedHashMap<>(); - final Map indicesHealths = new LinkedHashMap<>(); - final Map indicesStats = new LinkedHashMap<>(); + final Map indicesSettings = new LinkedHashMap<>(); + final Map indicesMetadatas = new LinkedHashMap<>(); + final Map indicesHealths = new LinkedHashMap<>(); + final Map indicesStats = new LinkedHashMap<>(); + @Before + public void setup() { + final int numIndices = randomIntBetween(3, 20); for (int i = 0; i < numIndices; i++) { String indexName = "index-" + i; @@ -136,11 +141,59 @@ public void testBuildTable() { } } } + } + public void testBuildTable() { final RestIndicesAction action = new RestIndicesAction(); - final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetadatas); + final Table table = action.buildTable( + new FakeRestRequest(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas, + action.getTableIterator(new String[0], indicesSettings), + null + ); // now, verify the table is correct + assertNotNull(table); + + assertTableHeaders(table); + + assertThat(table.getRows().size(), equalTo(indicesMetadatas.size())); + assertTableRows(table); + } + + public void testBuildPaginatedTable() { + final RestIndicesAction action = new RestIndicesAction(); + final RestIndicesListAction indicesListAction = new RestIndicesListAction(); + List indicesList = new ArrayList<>(indicesMetadatas.keySet()); + // Using half of the indices from metadata list for a page + String[] indicesToBeQueried = indicesList.subList(0, indicesMetadatas.size() / 2).toArray(new String[0]); + PageToken pageToken = new PageToken("foo", "indices"); + final Table table = action.buildTable( + new FakeRestRequest(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas, + indicesListAction.getTableIterator(indicesToBeQueried, indicesSettings), + pageToken + ); + + // verifying table + assertNotNull(table); + assertTableHeaders(table); + assertNotNull(table.getPageToken()); + assertEquals(pageToken.getNextToken(), table.getPageToken().getNextToken()); + assertEquals(pageToken.getPaginatedEntity(), table.getPageToken().getPaginatedEntity()); + + // Table should only contain the indices present in indicesToBeQueried + assertThat(table.getRows().size(), equalTo(indicesMetadatas.size() / 2)); + assertTableRows(table); + } + + private void assertTableHeaders(Table table) { List headers = table.getHeaders(); assertThat(headers.get(0).value, equalTo("health")); assertThat(headers.get(1).value, equalTo("status")); @@ -148,9 +201,10 @@ public void testBuildTable() { assertThat(headers.get(3).value, equalTo("uuid")); assertThat(headers.get(4).value, equalTo("pri")); assertThat(headers.get(5).value, equalTo("rep")); + } + private void assertTableRows(Table table) { final List> rows = table.getRows(); - assertThat(rows.size(), equalTo(indicesMetadatas.size())); for (final List row : rows) { final String indexName = (String) row.get(2).value; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java index 8183cb1d3b910..a82e563d70273 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java @@ -37,6 +37,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.rest.AbstractRestChannel; import org.opensearch.rest.RestResponse; +import org.opensearch.rest.pagination.PageToken; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; import org.junit.Before; @@ -64,9 +65,14 @@ public class RestTableTests extends OpenSearchTestCase { private static final String ACCEPT = "Accept"; private static final String TEXT_PLAIN = "text/plain; charset=UTF-8"; private static final String TEXT_TABLE_BODY = "foo foo foo foo foo foo foo foo\n"; + private static final String PAGINATED_TEXT_TABLE_BODY = "foo foo foo foo foo foo foo foo\nnext_token foo\n"; private static final String JSON_TABLE_BODY = "[{\"bulk.foo\":\"foo\",\"bulk.bar\":\"foo\",\"aliasedBulk\":\"foo\"," + "\"aliasedSecondBulk\":\"foo\",\"unmatched\":\"foo\"," + "\"invalidAliasesBulk\":\"foo\",\"timestamp\":\"foo\",\"epoch\":\"foo\"}]"; + private static final String PAGINATED_JSON_TABLE_BODY = + "{\"next_token\":\"foo\",\"entities\":[{\"bulk.foo\":\"foo\",\"bulk.bar\":\"foo\",\"aliasedBulk\":\"foo\"," + + "\"aliasedSecondBulk\":\"foo\",\"unmatched\":\"foo\"," + + "\"invalidAliasesBulk\":\"foo\",\"timestamp\":\"foo\",\"epoch\":\"foo\"}]}"; private static final String YAML_TABLE_BODY = "---\n" + "- bulk.foo: \"foo\"\n" + " bulk.bar: \"foo\"\n" @@ -76,6 +82,17 @@ public class RestTableTests extends OpenSearchTestCase { + " invalidAliasesBulk: \"foo\"\n" + " timestamp: \"foo\"\n" + " epoch: \"foo\"\n"; + private static final String PAGINATED_YAML_TABLE_BODY = "---\n" + + "next_token: \"foo\"\n" + + "entities:\n" + + "- bulk.foo: \"foo\"\n" + + " bulk.bar: \"foo\"\n" + + " aliasedBulk: \"foo\"\n" + + " aliasedSecondBulk: \"foo\"\n" + + " unmatched: \"foo\"\n" + + " invalidAliasesBulk: \"foo\"\n" + + " timestamp: \"foo\"\n" + + " epoch: \"foo\"\n"; private Table table; private FakeRestRequest restRequest; @@ -83,20 +100,7 @@ public class RestTableTests extends OpenSearchTestCase { public void setup() { restRequest = new FakeRestRequest(); table = new Table(); - table.startHeaders(); - table.addCell("bulk.foo", "alias:f;desc:foo"); - table.addCell("bulk.bar", "alias:b;desc:bar"); - // should be matched as well due to the aliases - table.addCell("aliasedBulk", "alias:bulkWhatever;desc:bar"); - table.addCell("aliasedSecondBulk", "alias:foobar,bulkolicious,bulkotastic;desc:bar"); - // no match - table.addCell("unmatched", "alias:un.matched;desc:bar"); - // invalid alias - table.addCell("invalidAliasesBulk", "alias:,,,;desc:bar"); - // timestamp - table.addCell("timestamp", "alias:ts"); - table.addCell("epoch", "alias:t"); - table.endHeaders(); + addHeaders(table); } public void testThatDisplayHeadersSupportWildcards() throws Exception { @@ -121,10 +125,28 @@ public void testThatWeUseTheAcceptHeaderJson() throws Exception { assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)), APPLICATION_JSON, JSON_TABLE_BODY); } + public void testThatWeUseTheAcceptHeaderJsonForPaginatedTable() throws Exception { + assertResponse( + Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)), + APPLICATION_JSON, + PAGINATED_JSON_TABLE_BODY, + getPaginatedTable() + ); + } + public void testThatWeUseTheAcceptHeaderYaml() throws Exception { assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)), APPLICATION_YAML, YAML_TABLE_BODY); } + public void testThatWeUseTheAcceptHeaderYamlForPaginatedTable() throws Exception { + assertResponse( + Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)), + APPLICATION_YAML, + PAGINATED_YAML_TABLE_BODY, + getPaginatedTable() + ); + } + public void testThatWeUseTheAcceptHeaderSmile() throws Exception { assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_SMILE)), APPLICATION_SMILE); } @@ -137,6 +159,15 @@ public void testThatWeUseTheAcceptHeaderText() throws Exception { assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)), TEXT_PLAIN, TEXT_TABLE_BODY); } + public void testThatWeUseTheAcceptHeaderTextForPaginatedTable() throws Exception { + assertResponse( + Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)), + TEXT_PLAIN, + PAGINATED_TEXT_TABLE_BODY, + getPaginatedTable() + ); + } + public void testIgnoreContentType() throws Exception { assertResponse(Collections.singletonMap(CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)), TEXT_PLAIN, TEXT_TABLE_BODY); } @@ -261,6 +292,10 @@ public void testMultiSort() { } private RestResponse assertResponseContentType(Map> headers, String mediaType) throws Exception { + return assertResponseContentType(headers, mediaType, table); + } + + private RestResponse assertResponseContentType(Map> headers, String mediaType, Table table) throws Exception { FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).build(); table.startRow(); table.addCell("foo"); @@ -282,7 +317,11 @@ public void sendResponse(RestResponse response) {} } private void assertResponse(Map> headers, String mediaType, String body) throws Exception { - RestResponse response = assertResponseContentType(headers, mediaType); + assertResponse(headers, mediaType, body, table); + } + + private void assertResponse(Map> headers, String mediaType, String body, Table table) throws Exception { + RestResponse response = assertResponseContentType(headers, mediaType, table); assertThat(response.content().utf8ToString(), equalTo(body)); } @@ -294,4 +333,28 @@ private List getHeaderNames(List headers) { return headerNames; } + + private Table getPaginatedTable() { + PageToken pageToken = new PageToken("foo", "entities"); + Table paginatedTable = new Table(pageToken); + addHeaders(paginatedTable); + return paginatedTable; + } + + private void addHeaders(Table table) { + table.startHeaders(); + table.addCell("bulk.foo", "alias:f;desc:foo"); + table.addCell("bulk.bar", "alias:b;desc:bar"); + // should be matched as well due to the aliases + table.addCell("aliasedBulk", "alias:bulkWhatever;desc:bar"); + table.addCell("aliasedSecondBulk", "alias:foobar,bulkolicious,bulkotastic;desc:bar"); + // no match + table.addCell("unmatched", "alias:un.matched;desc:bar"); + // invalid alias + table.addCell("invalidAliasesBulk", "alias:,,,;desc:bar"); + // timestamp + table.addCell("timestamp", "alias:ts"); + table.addCell("epoch", "alias:t"); + table.endHeaders(); + } } diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java new file mode 100644 index 0000000000000..01464b489e26e --- /dev/null +++ b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java @@ -0,0 +1,399 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.test.OpenSearchTestCase; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; +import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; +import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; + +public class IndexPaginationStrategyTests extends OpenSearchTestCase { + + public void testRetrieveAllIndicesWithVaryingPageSize() { + List indexNumberList = new ArrayList<>(); + final int totalIndices = 100; + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + // creating a cluster state with 100 indices + Collections.shuffle(indexNumberList, getRandom()); + ClusterState clusterState = getRandomClusterState(indexNumberList); + + // Checking pagination response for different pageSizes, which has a mix of even and odd numbers + // to ensure number of indices in last page is not always equal to pageSize. + List pageSizeList = List.of(1, 6, 10, 13); + List sortOrderList = List.of(PARAM_ASC_SORT_VALUE, PARAM_DESC_SORT_VALUE); + for (String sortOrder : sortOrderList) { + for (int pageSize : pageSizeList) { + String requestedToken = null; + int totalPagesToFetch = (int) Math.ceil(totalIndices / (pageSize * 1.0)); + int indicesRemaining = totalIndices; + for (int pageNumber = 1; pageNumber <= totalPagesToFetch; pageNumber++) { + PageParams pageParams = new PageParams(requestedToken, sortOrder, pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + if (pageNumber < totalPagesToFetch) { + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); + } else { + assertNull(paginationStrategy.getResponseToken().getNextToken()); + } + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // Asserting all the indices received + int responseItr = 0; + if (PARAM_ASC_SORT_VALUE.equals(sortOrder)) { + for (int indexNumber = (pageNumber - 1) * pageSize; indexNumber < Math.min( + 100, + pageNumber * pageSize + ); indexNumber++) { + assertEquals("test-index-" + (indexNumber + 1), paginationStrategy.getRequestedEntities().get(responseItr)); + responseItr++; + } + } else { + int endIndexNumberForPage = Math.max(indicesRemaining - pageSize, 0); + for (; indicesRemaining > endIndexNumberForPage; indicesRemaining--) { + assertEquals("test-index-" + indicesRemaining, paginationStrategy.getRequestedEntities().get(responseItr)); + responseItr++; + } + } + assertEquals(responseItr, paginationStrategy.getRequestedEntities().size()); + } + } + } + } + + public void testRetrieveAllIndicesInAscOrderWhileIndicesGetCreatedAndDeleted() { + List indexNumberList = new ArrayList<>(); + List deletedIndices = new ArrayList<>(); + final int totalIndices = 100; + final int numIndicesToDelete = 10; + final int numIndicesToCreate = 5; + List indicesFetched = new ArrayList<>(); + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + ClusterState clusterState = getRandomClusterState(indexNumberList); + + int pageSize = 6; + String requestedToken = null; + int numPages = 0; + do { + numPages++; + PageParams pageParams = new PageParams(requestedToken, PARAM_ASC_SORT_VALUE, pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertNotNull(paginationStrategy); + assertNotNull(paginationStrategy.getResponseToken()); + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // randomly deleting 10 indices after 3rd call + if (numPages == 3) { + deletedIndices = indexNumberList.subList(20, indexNumberList.size()); + Collections.shuffle(deletedIndices, getRandom()); + for (int pos = 0; pos < numIndicesToDelete; pos++) { + clusterState = deleteIndexFromClusterState(clusterState, deletedIndices.get(pos)); + } + } + // creating 5 indices after 5th call + if (numPages == 5) { + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + clusterState = addIndexToClusterState(clusterState, indexNumber); + } + } + if (requestedToken == null) { + assertEquals(paginationStrategy.getRequestedEntities().size(), 5); + } else { + assertEquals(paginationStrategy.getRequestedEntities().size(), pageSize); + } + + indicesFetched.addAll(paginationStrategy.getRequestedEntities()); + } while (Objects.nonNull(requestedToken)); + + assertEquals((int) Math.ceil((double) (totalIndices + numIndicesToCreate - numIndicesToDelete) / pageSize), numPages); + assertEquals(totalIndices + numIndicesToCreate - numIndicesToDelete, indicesFetched.size()); + + // none of the deleted index should appear in the list of fetched indices + for (int deletedIndexPos = 0; deletedIndexPos < numIndicesToDelete; deletedIndexPos++) { + assertFalse(indicesFetched.contains("test-index-" + deletedIndices.get(deletedIndexPos))); + } + + // all the newly created indices should be present in the list of fetched indices + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + assertTrue(indicesFetched.contains("test-index-" + indexNumber)); + } + } + + public void testRetrieveAllIndicesInDescOrderWhileIndicesGetCreatedAndDeleted() { + List indexNumberList = new ArrayList<>(); + List deletedIndices = new ArrayList<>(); + final int totalIndices = 100; + final int numIndicesToDelete = 9; + final int numIndicesToCreate = 5; + List indicesFetched = new ArrayList<>(); + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + ClusterState clusterState = getRandomClusterState(indexNumberList); + + int pageSize = 6; + String requestedToken = null; + int numPages = 0; + do { + numPages++; + PageParams pageParams = new PageParams(requestedToken, PARAM_DESC_SORT_VALUE, pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertNotNull(paginationStrategy); + assertNotNull(paginationStrategy.getResponseToken()); + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // randomly deleting 10 indices after 3rd call + if (numPages == 3) { + deletedIndices = indexNumberList.subList(0, 80); + Collections.shuffle(deletedIndices, getRandom()); + for (int pos = 0; pos < numIndicesToDelete; pos++) { + clusterState = deleteIndexFromClusterState(clusterState, deletedIndices.get(pos)); + } + } + // creating 5 indices after 5th call + if (numPages == 5) { + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + clusterState = addIndexToClusterState(clusterState, indexNumber); + } + } + if (requestedToken == null) { + assertEquals(paginationStrategy.getRequestedEntities().size(), (totalIndices - numIndicesToDelete) % pageSize); + } else { + assertEquals(paginationStrategy.getRequestedEntities().size(), pageSize); + } + + indicesFetched.addAll(paginationStrategy.getRequestedEntities()); + } while (Objects.nonNull(requestedToken)); + + assertEquals((int) Math.ceil((double) (totalIndices - numIndicesToDelete) / pageSize), numPages); + assertEquals(totalIndices - numIndicesToDelete, indicesFetched.size()); + + // none of the deleted index should appear in the list of fetched indices + for (int deletedIndexPos = 0; deletedIndexPos < numIndicesToDelete; deletedIndexPos++) { + assertFalse(indicesFetched.contains("test-index-" + deletedIndices.get(deletedIndexPos))); + } + + // none of the newly created indices should be present in the list of fetched indices + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + assertFalse(indicesFetched.contains("test-index-" + indexNumber)); + } + } + + public void testRetrieveIndicesWithSizeOneAndCurrentIndexGetsDeletedAscOrder() { + // Query1 with 4 indices in clusterState (test-index1,2,3,4) + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); + PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 1); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-1", paginationStrategy.getRequestedEntities().get(0)); + + // Adding index5 to clusterState, before executing next query. + clusterState = addIndexToClusterState(clusterState, 5); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); + + // Deleting test-index-2 which has already been displayed, still test-index-3 should get displayed + clusterState = deleteIndexFromClusterState(clusterState, 2); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0)); + + // Deleting test-index-4 which is not yet displayed which otherwise should have been displayed in the following query + // instead test-index-5 should now get displayed. + clusterState = deleteIndexFromClusterState(clusterState, 4); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, false); + assertEquals("test-index-5", paginationStrategy.getRequestedEntities().get(0)); + + } + + public void testRetrieveIndicesWithSizeOneAndCurrentIndexGetsDeletedDescOrder() { + // Query1 with 4 indices in clusterState (test-index1,2,3,4). + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); + PageParams pageParams = new PageParams(null, PARAM_DESC_SORT_VALUE, 1); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-4", paginationStrategy.getRequestedEntities().get(0)); + + // adding test-index-5 to clusterState, before executing next query. + clusterState = addIndexToClusterState(clusterState, 5); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0)); + + // Deleting test-index-3 which has already been displayed, still index2 should get displayed. + clusterState = deleteIndexFromClusterState(clusterState, 3); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); + + // Deleting test-index-1 which is not yet displayed which otherwise should have been displayed in the following query. + clusterState = deleteIndexFromClusterState(clusterState, 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 0, false); + } + + public void testRetrieveIndicesWithMultipleDeletionsAtOnceAscOrder() { + // Query1 with 5 indices in clusterState (test-index1,2,3,4,5). + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5)); + PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 1); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-1", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); + + // executing next query without any changes to clusterState + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); + + // Deleting test-index-1, test-index-2 & test-index-3 and executing next query. test-index-4 should get displayed. + clusterState = deleteIndexFromClusterState(clusterState, 1); + clusterState = deleteIndexFromClusterState(clusterState, 2); + clusterState = deleteIndexFromClusterState(clusterState, 3); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-4", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); + + // Executing the last query without any further change. Should result in test-index-5 and nextToken as null. + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-5", paginationStrategy.getRequestedEntities().get(0)); + assertNull(paginationStrategy.getResponseToken().getNextToken()); + } + + public void testRetrieveIndicesWithTokenModifiedToQueryBeyondTotal() { + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); + PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 10); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(10, paginationStrategy.getRequestedEntities().size()); + assertNull(paginationStrategy.getResponseToken().getNextToken()); + // creating a token with last sent index as test-index-10 + String token = clusterState.metadata().indices().get("test-index-10").getCreationDate() + "|" + "test-index-10"; + pageParams = new PageParams(Base64.getEncoder().encodeToString(token.getBytes(UTF_8)), PARAM_ASC_SORT_VALUE, 10); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(0, paginationStrategy.getRequestedEntities().size()); + assertNull(paginationStrategy.getResponseToken().getNextToken()); + } + + public void testCreatingIndexStrategyPageTokenWithRequestedTokenNull() { + try { + new IndexPaginationStrategy.IndexStrategyToken(null); + fail("expected exception"); + } catch (Exception e) { + assert e.getMessage().contains("requestedTokenString can not be null"); + } + } + + public void testIndexStrategyPageTokenWithWronglyEncryptedRequestToken() { + assertThrows(OpenSearchParseException.class, () -> new IndexPaginationStrategy.IndexStrategyToken("3%4%5")); + } + + public void testIndexStrategyPageTokenWithIncorrectNumberOfElementsInRequestedToken() { + assertThrows( + OpenSearchParseException.class, + () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1725361543")) + ); + assertThrows( + OpenSearchParseException.class, + () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1|1725361543|index|12345")) + ); + } + + public void testIndexStrategyPageTokenWithInvalidValuesInRequestedToken() { + assertThrows( + OpenSearchParseException.class, + () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("-1725361543|index")) + ); + } + + public void testCreatingIndexStrategyPageTokenWithNameOfLastRespondedIndexNull() { + try { + new IndexPaginationStrategy.IndexStrategyToken(1234l, null); + fail("expected exception"); + } catch (Exception e) { + assert e.getMessage().contains("index name should be provided"); + } + } + + /** + * @param indexNumbers would be used to create indices having names with integer appended after foo, like foo1, foo2. + * @return random clusterState consisting of indices having their creation times set to the integer used to name them. + */ + private ClusterState getRandomClusterState(List indexNumbers) { + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().build()) + .routingTable(RoutingTable.builder().build()) + .build(); + for (Integer indexNumber : indexNumbers) { + clusterState = addIndexToClusterState(clusterState, indexNumber); + } + return clusterState; + } + + private ClusterState addIndexToClusterState(ClusterState clusterState, int indexNumber) { + IndexMetadata indexMetadata = IndexMetadata.builder("test-index-" + indexNumber) + .settings( + settings(Version.CURRENT).put(SETTING_CREATION_DATE, Instant.now().plus(indexNumber, ChronoUnit.SECONDS).toEpochMilli()) + ) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + IndexRoutingTable.Builder indexRoutingTableBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex()); + return ClusterState.builder(clusterState) + .metadata(Metadata.builder(clusterState.metadata()).put(indexMetadata, true).build()) + .routingTable(RoutingTable.builder(clusterState.routingTable()).add(indexRoutingTableBuilder).build()) + .build(); + } + + private ClusterState deleteIndexFromClusterState(ClusterState clusterState, int indexNumber) { + return ClusterState.builder(clusterState) + .metadata(Metadata.builder(clusterState.metadata()).remove("test-index-" + indexNumber)) + .routingTable(RoutingTable.builder(clusterState.routingTable()).remove("test-index-" + indexNumber).build()) + .build(); + } + + private void assertPaginationResult(IndexPaginationStrategy paginationStrategy, int expectedEntities, boolean tokenExpected) { + assertNotNull(paginationStrategy); + assertEquals(expectedEntities, paginationStrategy.getRequestedEntities().size()); + assertNotNull(paginationStrategy.getResponseToken()); + assertEquals(tokenExpected, Objects.nonNull(paginationStrategy.getResponseToken().getNextToken())); + } + +} From 1bddf2f7255cf5facf17568a631cb605d78058a1 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Mon, 30 Sep 2024 10:29:41 +0530 Subject: [PATCH 073/108] Bugfix in RemoteFsTimestampAwareTranslog.trimUnreferencedReaders (#16078) Signed-off-by: Sachin Kale --- ...rePinnedTimestampsGarbageCollectionIT.java | 75 +++++++++++++ .../snapshots/DeleteSnapshotV2IT.java | 102 ++++++++++++++++++ .../RemoteFsTimestampAwareTranslog.java | 2 +- .../index/translog/RemoteFsTranslog.java | 8 ++ 4 files changed, 186 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java index 0a2668c60d3bd..08ece7df457cc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java @@ -8,6 +8,7 @@ package org.opensearch.remotestore; +import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; @@ -32,6 +33,7 @@ import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStorePinnedTimestampsGarbageCollectionIT extends RemoteStoreBaseIntegTestCase { @@ -288,6 +290,79 @@ public void testLiveIndexWithPinnedTimestamps() throws Exception { }); } + public void testLiveIndexWithPinnedTimestampsMultiplePrimaryTerms() throws Exception { + prepareCluster(1, 2, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(1, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 3) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + if (i == 2) { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueMinutes(1)); + remoteStorePinnedTimestampService.pinTimestamp(System.currentTimeMillis(), "xyz", noOpActionListener); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + } + } + + ingestDocs(); + + internalCluster().restartNode(primaryNodeName(INDEX_NAME)); + ensureGreen(INDEX_NAME); + + ingestDocs(); + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + + assertBusy(() -> { + List dataFiles = Files.list(translogDataPath).collect(Collectors.toList()); + assertFalse(dataFiles.isEmpty()); + }); + } + + private void ingestDocs() { + int numDocs = randomIntBetween(15, 20); + for (int i = 0; i < numDocs; i++) { + indexSingleDoc(INDEX_NAME, false); + } + + assertNoFailures(client().admin().indices().prepareRefresh(INDEX_NAME).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get()); + flushAndRefresh(INDEX_NAME); + + int numDocsPostFailover = randomIntBetween(15, 20); + for (int i = 0; i < numDocsPostFailover; i++) { + indexSingleDoc(INDEX_NAME, false); + } + + flushAndRefresh(INDEX_NAME); + assertNoFailures(client().admin().indices().prepareRefresh(INDEX_NAME).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get()); + } + public void testIndexDeletionNoPinnedTimestamps() throws Exception { prepareCluster(1, 1, Settings.EMPTY); Settings indexSettings = Settings.builder() diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java index c4e3a478c8540..7b2ad2bccd2b1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java @@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -312,6 +313,107 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2MultipleSnapshots( // translogPostDeletionOfSnapshot1.size()), 60, TimeUnit.SECONDS); } + public void testRemoteStoreCleanupMultiplePrimaryOnSnapshotDeletion() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + final Path remoteStoreRepoPath = randomRepoPath(); + Settings settings = remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath); + settings = Settings.builder() + .put(settings) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.FIXED.toString()) + .build(); + String clusterManagerName = internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(3, settings); + final Client clusterManagerClient = internalCluster().clusterManagerClient(); + ensureStableCluster(4); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + clusterManagerName + ); + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowV2(snapshotRepoPath)); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = Settings.builder() + .put(getRemoteStoreBackedIndexSettings()) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 2) + .build(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + ensureGreen(remoteStoreEnabledIndexName); + + // Create 2 snapshots for primary term 1 + keepPinnedTimestampSchedulerUpdated(); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + createSnapshot(snapshotRepoName, "snap1"); + keepPinnedTimestampSchedulerUpdated(); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + createSnapshot(snapshotRepoName, "snap2"); + + // Restart current primary to change the primary term + internalCluster().restartNode(primaryNodeName(remoteStoreEnabledIndexName)); + ensureGreen(remoteStoreEnabledIndexName); + + // Create 2 snapshots for primary term 2 + keepPinnedTimestampSchedulerUpdated(); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + createSnapshot(snapshotRepoName, "snap3"); + keepPinnedTimestampSchedulerUpdated(); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + createSnapshot(snapshotRepoName, "snap4"); + + String indexUUID = client().admin() + .indices() + .prepareGetSettings(remoteStoreEnabledIndexName) + .get() + .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_INDEX_UUID); + + Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID); + Path shardPath = Path.of(String.valueOf(indexPath), "0"); + Path translogPath = Path.of(String.valueOf(shardPath), "translog", "data", "1"); + + // Deleting snap1 will still keep files in primary term 1 due to snap2 + deleteSnapshot(clusterManagerClient, snapshotRepoName, "snap1"); + assertTrue(RemoteStoreBaseIntegTestCase.getFileCount(translogPath) > 0); + + // Deleting snap2 will not remove primary term 1 as we need to trigger trimUnreferencedReaders once + deleteSnapshot(clusterManagerClient, snapshotRepoName, "snap2"); + assertTrue(RemoteStoreBaseIntegTestCase.getFileCount(translogPath) > 0); + + // Index a doc to trigger trimUnreferencedReaders + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + keepPinnedTimestampSchedulerUpdated(); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + + assertBusy(() -> assertFalse(Files.exists(translogPath)), 30, TimeUnit.SECONDS); + } + + private void createSnapshot(String repoName, String snapshotName) { + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .get(); + SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName)); + } + + private void deleteSnapshot(Client clusterManagerClient, String repoName, String snapshotName) { + AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin() + .cluster() + .prepareDeleteSnapshot(repoName, snapshotName) + .get(); + assertAcked(deleteSnapshotResponse); + } + private Settings snapshotV2Settings(Path remoteStoreRepoPath) { Settings settings = Settings.builder() .put(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index e61a9606175ee..ede422ea3c4f7 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -121,7 +121,7 @@ public void trimUnreferencedReaders() throws IOException { protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal) throws IOException { if (trimLocal) { // clean up local translog files and updates readers - super.trimUnreferencedReaders(); + super.trimUnreferencedReaders(true); } // Update file tracker to reflect local translog state diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 80760ed6ddf12..f5a9ed8ed9362 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -556,9 +556,17 @@ protected Releasable drainSync() { @Override public void trimUnreferencedReaders() throws IOException { + trimUnreferencedReaders(false); + } + + protected void trimUnreferencedReaders(boolean onlyTrimLocal) throws IOException { // clean up local translog files and updates readers super.trimUnreferencedReaders(); + if (onlyTrimLocal) { + return; + } + // This is to ensure that after the permits are acquired during primary relocation, there are no further modification on remote // store. if (startedPrimarySupplier.getAsBoolean() == false || pauseSync.get()) { From 7ba8b78bfd4a3d97876f5fe34c354eed3ad98954 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Mon, 30 Sep 2024 13:47:55 +0530 Subject: [PATCH 074/108] Correct the version post backport to 2.18 instead of current (#16127) Signed-off-by: Gaurav Bafna --- .../main/java/org/opensearch/cluster/SnapshotsInProgress.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index 595f6d54fd9be..e3f98a3f61534 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -412,7 +412,7 @@ private Entry(StreamInput in) throws IOException { } else { remoteStoreIndexShallowCopy = false; } - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { remoteStoreIndexShallowCopyV2 = in.readBoolean(); } else { remoteStoreIndexShallowCopyV2 = false; @@ -886,7 +886,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeBoolean(remoteStoreIndexShallowCopy); } - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeBoolean(remoteStoreIndexShallowCopyV2); } } From 8ddb3eeee00978913fedd5aa49a84e718538ddae Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar <50844303+rahulkarajgikar@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:15:15 +0530 Subject: [PATCH 075/108] Add success and failure count OTel metrics for async shard fetch (#15976) Signed-off-by: Rahul Karajgikar --- CHANGELOG.md | 1 + .../TransportIndicesShardStoresAction.java | 27 ++- .../cluster/ClusterManagerMetrics.java | 13 + .../org/opensearch/cluster/ClusterModule.java | 3 + .../gateway/AsyncShardBatchFetch.java | 12 +- .../opensearch/gateway/AsyncShardFetch.java | 10 +- .../gateway/AsyncShardFetchCache.java | 7 +- .../opensearch/gateway/GatewayAllocator.java | 27 ++- .../gateway/ShardsBatchGatewayAllocator.java | 52 +++- .../gateway/AsyncShardFetchTests.java | 228 +++++++++++++++++- .../gateway/ShardBatchCacheTests.java | 5 +- .../snapshots/SnapshotResiliencyTests.java | 3 +- 12 files changed, 344 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b628e9277959d..9a45bbaa879fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) +- Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java index a8b97d0f344ae..1a3c657f5b1b8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java @@ -37,6 +37,7 @@ import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeReadAction; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; @@ -88,6 +89,7 @@ public class TransportIndicesShardStoresAction extends TransportClusterManagerNo private static final Logger logger = LogManager.getLogger(TransportIndicesShardStoresAction.class); private final TransportNodesListGatewayStartedShards listShardStoresInfo; + private final ClusterManagerMetrics clusterManagerMetrics; @Inject public TransportIndicesShardStoresAction( @@ -96,7 +98,8 @@ public TransportIndicesShardStoresAction( ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - TransportNodesListGatewayStartedShards listShardStoresInfo + TransportNodesListGatewayStartedShards listShardStoresInfo, + ClusterManagerMetrics clusterManagerMetrics ) { super( IndicesShardStoresAction.NAME, @@ -109,6 +112,7 @@ public TransportIndicesShardStoresAction( true ); this.listShardStoresInfo = listShardStoresInfo; + this.clusterManagerMetrics = clusterManagerMetrics; } @Override @@ -154,7 +158,7 @@ protected void clusterManagerOperation( // we could fetch all shard store info from every node once (nNodes requests) // we have to implement a TransportNodesAction instead of using TransportNodesListGatewayStartedShards // for fetching shard stores info, that operates on a list of shards instead of a single shard - new AsyncShardStoresInfoFetches(state.nodes(), routingNodes, shardsToFetch, listener).start(); + new AsyncShardStoresInfoFetches(state.nodes(), routingNodes, shardsToFetch, listener, clusterManagerMetrics).start(); } @Override @@ -175,12 +179,14 @@ private class AsyncShardStoresInfoFetches { private final ActionListener listener; private CountDown expectedOps; private final Queue fetchResponses; + private final ClusterManagerMetrics clusterManagerMetrics; AsyncShardStoresInfoFetches( DiscoveryNodes nodes, RoutingNodes routingNodes, Set> shards, - ActionListener listener + ActionListener listener, + ClusterManagerMetrics clusterManagerMetrics ) { this.nodes = nodes; this.routingNodes = routingNodes; @@ -188,6 +194,7 @@ private class AsyncShardStoresInfoFetches { this.listener = listener; this.fetchResponses = new ConcurrentLinkedQueue<>(); this.expectedOps = new CountDown(shards.size()); + this.clusterManagerMetrics = clusterManagerMetrics; } void start() { @@ -195,7 +202,14 @@ void start() { listener.onResponse(new IndicesShardStoresResponse()); } else { for (Tuple shard : shards) { - InternalAsyncFetch fetch = new InternalAsyncFetch(logger, "shard_stores", shard.v1(), shard.v2(), listShardStoresInfo); + InternalAsyncFetch fetch = new InternalAsyncFetch( + logger, + "shard_stores", + shard.v1(), + shard.v2(), + listShardStoresInfo, + clusterManagerMetrics + ); fetch.fetchData(nodes, Collections.emptyMap()); } } @@ -213,9 +227,10 @@ private class InternalAsyncFetch extends AsyncShardFetch deciderList; final ShardsAllocator shardsAllocator; + private final ClusterManagerMetrics clusterManagerMetrics; public ClusterModule( Settings settings, @@ -166,6 +167,7 @@ public ClusterModule( settings, clusterManagerMetrics ); + this.clusterManagerMetrics = clusterManagerMetrics; } public static List getNamedWriteables() { @@ -456,6 +458,7 @@ protected void configure() { bind(TaskResultsService.class).asEagerSingleton(); bind(AllocationDeciders.class).toInstance(allocationDeciders); bind(ShardsAllocator.class).toInstance(shardsAllocator); + bind(ClusterManagerMetrics.class).toInstance(clusterManagerMetrics); } public void setExistingShardsAllocators(GatewayAllocator gatewayAllocator, ShardsBatchGatewayAllocator shardsBatchGatewayAllocator) { diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java index df642a9f5a743..d86d41bb1a359 100644 --- a/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java +++ b/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.logging.Loggers; import org.opensearch.core.index.shard.ShardId; @@ -48,7 +49,8 @@ public abstract class AsyncShardBatchFetch extend Class clazz, V emptyShardResponse, Predicate emptyShardResponsePredicate, - ShardBatchResponseFactory responseFactory + ShardBatchResponseFactory responseFactory, + ClusterManagerMetrics clusterManagerMetrics ) { super( logger, @@ -64,7 +66,8 @@ public abstract class AsyncShardBatchFetch extend clazz, emptyShardResponse, emptyShardResponsePredicate, - responseFactory + responseFactory, + clusterManagerMetrics ) ); } @@ -116,9 +119,10 @@ public ShardBatchCache( Class clazz, V emptyResponse, Predicate emptyShardResponsePredicate, - ShardBatchResponseFactory responseFactory + ShardBatchResponseFactory responseFactory, + ClusterManagerMetrics clusterManagerMetrics ) { - super(Loggers.getLogger(logger, "_" + logKey), type); + super(Loggers.getLogger(logger, "_" + logKey), type, clusterManagerMetrics); this.batchSize = shardAttributesMap.size(); this.emptyShardResponsePredicate = emptyShardResponsePredicate; cache = new HashMap<>(); diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java index b664dd573ce67..6017743ef2bd0 100644 --- a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java +++ b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java @@ -35,6 +35,7 @@ import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.allocation.RoutingAllocation; @@ -94,7 +95,8 @@ protected AsyncShardFetch( String type, ShardId shardId, String customDataPath, - Lister, T> action + Lister, T> action, + ClusterManagerMetrics clusterManagerMetrics ) { this.logger = logger; this.type = type; @@ -102,7 +104,7 @@ protected AsyncShardFetch( shardAttributesMap.put(shardId, new ShardAttributes(customDataPath)); this.action = (Lister, T>) action; this.reroutingKey = "ShardId=[" + shardId.toString() + "]"; - cache = new ShardCache<>(logger, reroutingKey, type); + cache = new ShardCache<>(logger, reroutingKey, type, clusterManagerMetrics); } /** @@ -284,8 +286,8 @@ static class ShardCache extends AsyncShardFetchCache private final Map> cache; - public ShardCache(Logger logger, String logKey, String type) { - super(Loggers.getLogger(logger, "_" + logKey), type); + public ShardCache(Logger logger, String logKey, String type, ClusterManagerMetrics clusterManagerMetrics) { + super(Loggers.getLogger(logger, "_" + logKey), type, clusterManagerMetrics); cache = new HashMap<>(); } diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java b/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java index 2a4e6181467b0..9b0a95f611e0e 100644 --- a/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java +++ b/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java @@ -14,6 +14,7 @@ import org.opensearch.OpenSearchTimeoutException; import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.nodes.BaseNodeResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; @@ -51,10 +52,12 @@ public abstract class AsyncShardFetchCache { private final Logger logger; private final String type; + private final ClusterManagerMetrics clusterManagerMetrics; - protected AsyncShardFetchCache(Logger logger, String type) { + protected AsyncShardFetchCache(Logger logger, String type, ClusterManagerMetrics clusterManagerMetrics) { this.logger = logger; this.type = type; + this.clusterManagerMetrics = clusterManagerMetrics; } abstract void initData(DiscoveryNode node); @@ -162,6 +165,7 @@ Map getCacheData(DiscoveryNodes nodes, Set failedNodes } void processResponses(List responses, long fetchingRound) { + clusterManagerMetrics.incrementCounter(clusterManagerMetrics.asyncFetchSuccessCounter, Double.valueOf(responses.size())); for (K response : responses) { BaseNodeEntry nodeEntry = getCache().get(response.getNode().getId()); if (nodeEntry != null) { @@ -222,6 +226,7 @@ boolean retryableException(Throwable unwrappedCause) { } void processFailures(List failures, long fetchingRound) { + clusterManagerMetrics.incrementCounter(clusterManagerMetrics.asyncFetchFailureCounter, Double.valueOf(failures.size())); for (FailedNodeException failure : failures) { logger.trace("processing failure {} for [{}]", failure, type); BaseNodeEntry nodeEntry = getCache().get(failure.nodeId()); diff --git a/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java index c8ef9364ebba9..eaacb5dbfbd17 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java @@ -37,6 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -92,11 +93,12 @@ public class GatewayAllocator implements ExistingShardsAllocator { public GatewayAllocator( RerouteService rerouteService, TransportNodesListGatewayStartedShards startedAction, - TransportNodesListShardStoreMetadata storeAction + TransportNodesListShardStoreMetadata storeAction, + ClusterManagerMetrics clusterManagerMetrics ) { this.rerouteService = rerouteService; - this.primaryShardAllocator = new InternalPrimaryShardAllocator(startedAction); - this.replicaShardAllocator = new InternalReplicaShardAllocator(storeAction); + this.primaryShardAllocator = new InternalPrimaryShardAllocator(startedAction, clusterManagerMetrics); + this.replicaShardAllocator = new InternalReplicaShardAllocator(storeAction, clusterManagerMetrics); } @Override @@ -251,9 +253,10 @@ class InternalAsyncFetch extends AsyncShardFetch String type, ShardId shardId, String customDataPath, - Lister, T> action + Lister, T> action, + ClusterManagerMetrics clusterManagerMetrics ) { - super(logger, type, shardId, customDataPath, action); + super(logger, type, shardId, customDataPath, action, clusterManagerMetrics); } @Override @@ -274,9 +277,11 @@ protected void reroute(String reroutingKey, String reason) { class InternalPrimaryShardAllocator extends PrimaryShardAllocator { private final TransportNodesListGatewayStartedShards startedAction; + private final ClusterManagerMetrics clusterManagerMetrics; - InternalPrimaryShardAllocator(TransportNodesListGatewayStartedShards startedAction) { + InternalPrimaryShardAllocator(TransportNodesListGatewayStartedShards startedAction, ClusterManagerMetrics clusterManagerMetrics) { this.startedAction = startedAction; + this.clusterManagerMetrics = clusterManagerMetrics; } @Override @@ -291,7 +296,8 @@ protected AsyncShardFetch.FetchResult shardState = fetch.fetchData( @@ -313,9 +319,11 @@ protected AsyncShardFetch.FetchResult shardStores = fetch.fetchData( diff --git a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java index 5e2dcbcd70b40..d7c0a66ba3424 100644 --- a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java @@ -13,6 +13,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -44,6 +45,7 @@ import org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadata; import org.opensearch.indices.store.TransportNodesListShardStoreMetadataHelper; import org.opensearch.indices.store.TransportNodesListShardStoreMetadataHelper.StoreFilesMetadata; +import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; import java.util.ArrayList; import java.util.Collections; @@ -81,6 +83,7 @@ public class ShardsBatchGatewayAllocator implements ExistingShardsAllocator { private TimeValue primaryShardsBatchGatewayAllocatorTimeout; private TimeValue replicaShardsBatchGatewayAllocatorTimeout; public static final TimeValue MIN_ALLOCATOR_TIMEOUT = TimeValue.timeValueSeconds(20); + private final ClusterManagerMetrics clusterManagerMetrics; /** * Number of shards we send in one batch to data nodes for fetching metadata @@ -160,7 +163,8 @@ public ShardsBatchGatewayAllocator( TransportNodesListGatewayStartedShardsBatch batchStartedAction, TransportNodesListShardStoreMetadataBatch batchStoreAction, Settings settings, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + ClusterManagerMetrics clusterManagerMetrics ) { this.rerouteService = rerouteService; this.primaryShardBatchAllocator = new InternalPrimaryBatchShardAllocator(); @@ -172,6 +176,7 @@ public ShardsBatchGatewayAllocator( clusterSettings.addSettingsUpdateConsumer(PRIMARY_BATCH_ALLOCATOR_TIMEOUT_SETTING, this::setPrimaryBatchAllocatorTimeout); this.replicaShardsBatchGatewayAllocatorTimeout = REPLICA_BATCH_ALLOCATOR_TIMEOUT_SETTING.get(settings); clusterSettings.addSettingsUpdateConsumer(REPLICA_BATCH_ALLOCATOR_TIMEOUT_SETTING, this::setReplicaBatchAllocatorTimeout); + this.clusterManagerMetrics = clusterManagerMetrics; } @Override @@ -187,6 +192,7 @@ protected ShardsBatchGatewayAllocator() { this(DEFAULT_SHARD_BATCH_SIZE, null); } + // for tests protected ShardsBatchGatewayAllocator(long batchSize, RerouteService rerouteService) { this.rerouteService = rerouteService; this.batchStartedAction = null; @@ -196,10 +202,9 @@ protected ShardsBatchGatewayAllocator(long batchSize, RerouteService rerouteServ this.maxBatchSize = batchSize; this.primaryShardsBatchGatewayAllocatorTimeout = null; this.replicaShardsBatchGatewayAllocatorTimeout = null; + this.clusterManagerMetrics = new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE); } - // for tests - @Override public int getNumberOfInFlightFetches() { int count = 0; @@ -413,7 +418,7 @@ else if (shardRouting.primary() == primary) { // add to batch if batch size full or last shard in unassigned list if (batchSize == 0 || iterator.hasNext() == false) { String batchUUId = UUIDs.base64UUID(); - ShardsBatch shardsBatch = new ShardsBatch(batchUUId, perBatchShards, primary); + ShardsBatch shardsBatch = new ShardsBatch(batchUUId, perBatchShards, primary, clusterManagerMetrics); // add the batch to list of current batches addBatch(shardsBatch, primary); batchesToBeAssigned.add(batchUUId); @@ -588,9 +593,21 @@ class InternalBatchAsyncFetch extends AsyncShardB Class clazz, V emptyShardResponse, Predicate emptyShardResponsePredicate, - ShardBatchResponseFactory responseFactory + ShardBatchResponseFactory responseFactory, + ClusterManagerMetrics clusterManagerMetrics ) { - super(logger, type, map, action, batchUUId, clazz, emptyShardResponse, emptyShardResponsePredicate, responseFactory); + super( + logger, + type, + map, + action, + batchUUId, + clazz, + emptyShardResponse, + emptyShardResponsePredicate, + responseFactory, + clusterManagerMetrics + ); } @Override @@ -650,16 +667,17 @@ protected boolean hasInitiatedFetching(ShardRouting shard) { * It should return false if there has never been a fetch for this batch. * This function is currently only used in the case of replica shards when all deciders returned NO/THROTTLE, and explain mode is ON. * Allocation explain and manual reroute APIs try to append shard store information (matching bytes) to the allocation decision. - * However, these APIs do not want to trigger a new asyncFetch for these ineligible shards, unless the data from nodes is already there. + * However, these APIs do not want to trigger a new asyncFetch for these ineligible shards + * They only want to use the data if it is already available. * This function is used to see if a fetch has happened to decide if it is possible to append shard store info without a new async fetch. * In the case when shard has a batch but no fetch has happened before, it would be because it is a new batch. * In the case when shard has a batch, and a fetch has happened before, and no fetch is ongoing, it would be because we have already completed fetch for all nodes. - * + *

* In order to check if a fetch has ever happened, we check 2 things: * 1. If the shard batch cache is empty, we know that fetch has never happened so we return false. * 2. If we see that the list of nodes to fetch from is empty, we know that all nodes have data or are ongoing a fetch. So we return true. * 3. Otherwise we return false. - * + *

* see {@link AsyncShardFetchCache#findNodesToFetch()} */ String batchId = getBatchId(shard, shard.primary()); @@ -669,7 +687,8 @@ protected boolean hasInitiatedFetching(ShardRouting shard) { logger.trace("Checking if fetching done for batch id {}", batchId); ShardsBatch shardsBatch = shard.primary() ? batchIdToStartedShardBatch.get(batchId) : batchIdToStoreShardBatch.get(batchId); // if fetchData has never been called, the per node cache will be empty and have no nodes - // this is because cache.fillShardCacheWithDataNodes(nodes) initialises this map and is called in AsyncShardFetch.fetchData + /// this is because {@link AsyncShardFetchCache#fillShardCacheWithDataNodes(DiscoveryNodes)} initialises this map + /// and is called in {@link AsyncShardFetch#fetchData(DiscoveryNodes, Map)} if (shardsBatch == null || shardsBatch.getAsyncFetcher().hasEmptyCache()) { logger.trace("Batch cache is empty for batch {} ", batchId); return false; @@ -739,7 +758,12 @@ public class ShardsBatch { private final Map batchInfo; - public ShardsBatch(String batchId, Map shardsWithInfo, boolean primary) { + public ShardsBatch( + String batchId, + Map shardsWithInfo, + boolean primary, + ClusterManagerMetrics clusterManagerMetrics + ) { this.batchId = batchId; this.batchInfo = new HashMap<>(shardsWithInfo); // create a ShardId -> customDataPath map for async fetch @@ -757,7 +781,8 @@ public ShardsBatch(String batchId, Map shardsWithInfo, bool GatewayStartedShard.class, new GatewayStartedShard(null, false, null, null), GatewayStartedShard::isEmpty, - new ShardBatchResponseFactory<>(true) + new ShardBatchResponseFactory<>(true), + clusterManagerMetrics ); } else { asyncBatch = new InternalBatchAsyncFetch<>( @@ -769,7 +794,8 @@ public ShardsBatch(String batchId, Map shardsWithInfo, bool NodeStoreFilesMetadata.class, new NodeStoreFilesMetadata(new StoreFilesMetadata(null, Store.MetadataSnapshot.EMPTY, Collections.emptyList()), null), NodeStoreFilesMetadata::isEmpty, - new ShardBatchResponseFactory<>(false) + new ShardBatchResponseFactory<>(false), + clusterManagerMetrics ); } } diff --git a/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java b/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java index db97c3ece94ba..c25150873a1ce 100644 --- a/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java +++ b/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java @@ -35,10 +35,13 @@ import org.opensearch.Version; import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.nodes.BaseNodeResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.telemetry.metrics.Counter; +import org.opensearch.telemetry.metrics.MetricsRegistry; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -54,6 +57,12 @@ import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.anyDouble; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class AsyncShardFetchTests extends OpenSearchTestCase { private final DiscoveryNode node1 = new DiscoveryNode( @@ -78,13 +87,29 @@ public class AsyncShardFetchTests extends OpenSearchTestCase { private ThreadPool threadPool; private TestFetch test; + private Counter asyncFetchSuccessCounter; + private Counter asyncFetchFailureCounter; + private Counter dummyCounter; @Override @Before public void setUp() throws Exception { super.setUp(); this.threadPool = new TestThreadPool(getTestName()); - this.test = new TestFetch(threadPool); + final MetricsRegistry metricsRegistry = mock(MetricsRegistry.class); + this.asyncFetchFailureCounter = mock(Counter.class); + this.asyncFetchSuccessCounter = mock(Counter.class); + this.dummyCounter = mock(Counter.class); + when(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer(invocationOnMock -> { + String counterName = (String) invocationOnMock.getArguments()[0]; + if (counterName.contains("async.fetch.success.count")) { + return asyncFetchSuccessCounter; + } else if (counterName.contains("async.fetch.failure.count")) { + return asyncFetchFailureCounter; + } + return dummyCounter; + }); + this.test = new TestFetch(threadPool, metricsRegistry); } @After @@ -100,14 +125,26 @@ public void testClose() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter remains 0 because fetch is ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire a response, wait on reroute incrementing test.fireSimulationAndWait(node1.getId()); + // counter goes up because fetch completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); + // verify we get back the data node assertThat(test.reroute.get(), equalTo(1)); test.close(); try { test.fetchData(nodes, emptyMap()); + // counter should not go up when calling fetchData since fetch never completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); fail("fetch data should fail when closed"); } catch (IllegalStateException e) { // all is well @@ -125,12 +162,21 @@ public void testFullCircleSingleNodeSuccess() throws Exception { // fire a response, wait on reroute incrementing test.fireSimulationAndWait(node1.getId()); + // total counter goes up by 1 after success + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); + // verify we get back the data node assertThat(test.reroute.get(), equalTo(1)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // counter remains same because fetchData does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); } public void testFullCircleSingleNodeFailure() throws Exception { @@ -145,24 +191,47 @@ public void testFullCircleSingleNodeFailure() throws Exception { // fire a response, wait on reroute incrementing test.fireSimulationAndWait(node1.getId()); + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); + // failure, fetched data exists, but has no data assertThat(test.reroute.get(), equalTo(1)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(0)); + // counter remains same because fetchData does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); // on failure, we reset the failure on a successive call to fetchData, and try again afterwards test.addSimulation(node1.getId(), response1); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // No additional failure, empty data so no change in counter + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); test.fireSimulationAndWait(node1.getId()); + // Success counter will increase + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); + // 2 reroutes, cause we have a failure that we clear assertThat(test.reroute.get(), equalTo(3)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // counter remains same because fetchData does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); } public void testIgnoreResponseFromDifferentRound() throws Exception { @@ -173,20 +242,40 @@ public void testIgnoreResponseFromDifferentRound() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because fetchData is not completed + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // handle a response with incorrect round id, wait on reroute incrementing test.processAsyncFetch(Collections.singletonList(response1), Collections.emptyList(), 0); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(1)); + // success counter increments to 1 because we called processAsyncFetch with a valid response, even though the round was incorrect + // failure counter also increments by 1 with empty list + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(0.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); // fire a response (with correct round id), wait on reroute incrementing test.fireSimulationAndWait(node1.getId()); + // success counter now goes up by 1 because fetchData completed + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(0.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); + // verify we get back the data node assertThat(test.reroute.get(), equalTo(2)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // total counter remains same because fetchdata does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(0.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); } public void testIgnoreFailureFromDifferentRound() throws Exception { @@ -198,6 +287,9 @@ public void testIgnoreFailureFromDifferentRound() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because fetchData still ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // handle a failure with incorrect round id, wait on reroute incrementing test.processAsyncFetch( @@ -207,14 +299,30 @@ public void testIgnoreFailureFromDifferentRound() throws Exception { ); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(1)); + // success counter called with empty list + // failure counter goes up by 1 because of the failure + verify(asyncFetchSuccessCounter, times(1)).add(0.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); // fire a response, wait on reroute incrementing test.fireSimulationAndWait(node1.getId()); + // failure counter goes up by 1 because of the failure + verify(asyncFetchSuccessCounter, times(1)).add(0.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(2)).add(1.0); + verify(asyncFetchFailureCounter, times(2)).add(anyDouble()); // failure, fetched data exists, but has no data assertThat(test.reroute.get(), equalTo(2)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(0)); + // counters remain same because fetchData does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(1)).add(0.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(2)).add(1.0); + verify(asyncFetchFailureCounter, times(2)).add(anyDouble()); } public void testTwoNodesOnSetup() throws Exception { @@ -226,16 +334,32 @@ public void testTwoNodesOnSetup() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because fetch ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the first response, it should trigger a reroute test.fireSimulationAndWait(node1.getId()); + // counter 1 because one fetch completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); + // there is still another on going request, so no data assertThat(test.getNumberOfInFlightFetches(), equalTo(1)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // counter still 1 because fetchData did not trigger new async fetch + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the second simulation, this should allow us to get the data test.fireSimulationAndWait(node2.getId()); + // counter 2 because 2 fetches completed + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // no more ongoing requests, we should fetch the data assertThat(test.reroute.get(), equalTo(2)); fetchData = test.fetchData(nodes, emptyMap()); @@ -243,6 +367,10 @@ public void testTwoNodesOnSetup() throws Exception { assertThat(fetchData.getData().size(), equalTo(2)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); assertThat(fetchData.getData().get(node2), sameInstance(response2)); + // counter still 2 because fetchData call did not trigger new async fetch + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); } public void testTwoNodesOnSetupAndFailure() throws Exception { @@ -254,34 +382,59 @@ public void testTwoNodesOnSetupAndFailure() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because both fetches ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the first response, it should trigger a reroute test.fireSimulationAndWait(node1.getId()); assertThat(test.reroute.get(), equalTo(1)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // counter 1 because one fetch completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the second simulation, this should allow us to get the data test.fireSimulationAndWait(node2.getId()); + // failure counter up by 1 because one fetch failed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); assertThat(test.reroute.get(), equalTo(2)); + // since one of those failed, we should only have one entry fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // success and failure counters same because fetchData did not trigger new async fetch + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); } public void testTwoNodesAddedInBetween() throws Exception { DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).build(); test.addSimulation(node1.getId(), response1); - // no fetched data, 2 requests still on going + // no fetched data, request still on going AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because both fetches ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the first response, it should trigger a reroute test.fireSimulationAndWait(node1.getId()); + // counter 1 because fetch completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // now, add a second node to the nodes, it should add it to the ongoing requests nodes = DiscoveryNodes.builder(nodes).add(node2).build(); @@ -289,16 +442,28 @@ public void testTwoNodesAddedInBetween() throws Exception { // no fetch data, has a new node introduced fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // counter still 1 because second fetch ongoing + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the second simulation, this should allow us to get the data test.fireSimulationAndWait(node2.getId()); + // counter now 2 because 2 fetches completed + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); - // since one of those failed, we should only have one entry + // since both succeeded, we should have 2 entries fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(2)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); assertThat(fetchData.getData().get(node2), sameInstance(response2)); + // counter still 2 because fetchData did not trigger new async fetch + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); } public void testClearCache() throws Exception { @@ -312,21 +477,36 @@ public void testClearCache() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because fetch ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); test.fireSimulationAndWait(node1.getId()); assertThat(test.reroute.get(), equalTo(1)); + // counter 1 because 1 fetch completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // verify we get back right data from node fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // counter still 1 because a new fetch is not called + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // second fetch gets same data fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // counter still 1 because a new fetch is not called + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); test.clearCacheForNode(node1.getId()); @@ -336,15 +516,27 @@ public void testClearCache() throws Exception { // no fetched data, new request on going fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // counter still 1 because new fetch is still ongoing + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); test.fireSimulationAndWait(node1.getId()); assertThat(test.reroute.get(), equalTo(2)); + // counter now 2 because second fetch completed + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // verify we get new data back fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1_2)); + // counter still 2 because fetchData did not trigger new async fetch + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); } public void testConcurrentRequestAndClearCache() throws Exception { @@ -355,12 +547,19 @@ public void testConcurrentRequestAndClearCache() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because fetch ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // clear cache while request is still on going, before it is processed test.clearCacheForNode(node1.getId()); test.fireSimulationAndWait(node1.getId()); assertThat(test.reroute.get(), equalTo(1)); + // counter 1 because fetch completed, even though cache was wiped + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // prepare next request test.addSimulation(node1.getId(), response1_2); @@ -368,15 +567,27 @@ public void testConcurrentRequestAndClearCache() throws Exception { // verify still no fetched data, request still on going fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // counter unchanged because fetch ongoing + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); test.fireSimulationAndWait(node1.getId()); assertThat(test.reroute.get(), equalTo(2)); + // counter 2 because second fetch completed + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // verify we get new data back fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1_2)); + // counter unchanged because fetchData does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); } @@ -398,8 +609,15 @@ static class Entry { private final Map simulations = new ConcurrentHashMap<>(); private AtomicInteger reroute = new AtomicInteger(); - TestFetch(ThreadPool threadPool) { - super(LogManager.getLogger(TestFetch.class), "test", new ShardId("test", "_na_", 1), "", null); + TestFetch(ThreadPool threadPool, MetricsRegistry metricsRegistry) { + super( + LogManager.getLogger(TestFetch.class), + "test", + new ShardId("test", "_na_", 1), + "", + null, + new ClusterManagerMetrics(metricsRegistry) + ); this.threadPool = threadPool; } diff --git a/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java b/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java index 12030ad41d508..39c4ee8c8ca06 100644 --- a/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java +++ b/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java @@ -8,6 +8,7 @@ package org.opensearch.gateway; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.OpenSearchAllocationTestCase; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -19,6 +20,7 @@ import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.GatewayStartedShard; import org.opensearch.gateway.TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch; import org.opensearch.indices.store.ShardAttributes; +import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; import java.util.ArrayList; import java.util.HashMap; @@ -52,7 +54,8 @@ public void setupShardBatchCache(String batchId, int numberOfShards) { GatewayStartedShard.class, new GatewayStartedShard(null, false, null, null), GatewayStartedShard::isEmpty, - new ShardBatchResponseFactory<>(true) + new ShardBatchResponseFactory<>(true), + new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) ); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 440227436175d..d17e661615b0d 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2429,7 +2429,8 @@ public void onFailure(final Exception e) { nodeEnv, indicesService, namedXContentRegistry - ) + ), + new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) ) ); actions.put( From e6e290d7d77feec1404e296d130aff536c482392 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Mon, 30 Sep 2024 19:41:12 +0800 Subject: [PATCH 076/108] Fix version check in yml test file for double range profiler shows filter rewrite info (#16125) Signed-off-by: Gao Binlong --- .../rest-api-spec/test/search.aggregation/40_range.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml index 1e1d2b0706d6b..93ca668f93a6f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml @@ -535,8 +535,8 @@ setup: --- "Double range profiler shows filter rewrite info": - skip: - version: " - 2.99.99" - reason: debug info for filter rewrite added in 3.0.0 (to be backported to 2.15.0) + version: " - 2.15.99" + reason: debug info for filter rewrite was added in 2.16.0 - do: indices.create: From 0b96565b51326e916e9fe33ead44187844c23c46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:13:34 -0400 Subject: [PATCH 077/108] Bump com.maxmind.db:maxmind-db from 3.1.0 to 3.1.1 in /modules/ingest-geoip (#16137) * Bump com.maxmind.db:maxmind-db in /modules/ingest-geoip Bumps [com.maxmind.db:maxmind-db](https://github.com/maxmind/MaxMind-DB-Reader-java) from 3.1.0 to 3.1.1. - [Release notes](https://github.com/maxmind/MaxMind-DB-Reader-java/releases) - [Changelog](https://github.com/maxmind/MaxMind-DB-Reader-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/maxmind/MaxMind-DB-Reader-java/compare/v3.1.0...v3.1.1) --- updated-dependencies: - dependency-name: com.maxmind.db:maxmind-db dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + modules/ingest-geoip/build.gradle | 2 +- modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 | 1 - modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a45bbaa879fd..b01badbe3fd2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) - Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) +- Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 162ac6f9ad620..f74de1dc290dd 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -41,7 +41,7 @@ opensearchplugin { dependencies { api('com.maxmind.geoip2:geoip2:4.2.1') // geoip2 dependencies: - api('com.maxmind.db:maxmind-db:3.1.0') + api('com.maxmind.db:maxmind-db:3.1.1') api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}") api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}") diff --git a/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 deleted file mode 100644 index 9db7c7319af0b..0000000000000 --- a/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2008992ab45d61c7b28a18678b5df82272529da3 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 new file mode 100644 index 0000000000000..f1a25c6d600c0 --- /dev/null +++ b/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 @@ -0,0 +1 @@ +84a2a5d322fef1f1e82f05095683c8d53902baf1 \ No newline at end of file From 9a5c7b2eae405d08c4d5d64f0fef087821c18655 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Mon, 30 Sep 2024 22:16:52 +0530 Subject: [PATCH 078/108] Add default implementation to new finalizeSnapshot() in Repository (#16128) Signed-off-by: Sachin Kale --- .../main/java/org/opensearch/repositories/Repository.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index d4520beb5b570..138bc13140aea 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -180,7 +180,7 @@ void finalizeSnapshot( * @param repositoryUpdatePriority priority for the cluster state update task * @param listener listener to be invoked with the new {@link RepositoryData} after completing the snapshot */ - void finalizeSnapshot( + default void finalizeSnapshot( ShardGenerations shardGenerations, long repositoryStateId, Metadata clusterMetadata, @@ -189,7 +189,9 @@ void finalizeSnapshot( Function stateTransformer, Priority repositoryUpdatePriority, ActionListener listener - ); + ) { + throw new UnsupportedOperationException(); + } /** * Deletes snapshots From d131d58d842d18e62e523d5c6833d8c86652ce81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:24:41 -0400 Subject: [PATCH 079/108] Bump org.apache.logging.log4j:log4j-core from 2.24.0 to 2.24.1 in /buildSrc/src/testKit/thirdPartyAudit/sample_jars (#16134) * Bump org.apache.logging.log4j:log4j-core Bumps org.apache.logging.log4j:log4j-core from 2.24.0 to 2.24.1. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b01badbe3fd2e..da14948781849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) -- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) +- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.1 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858), [#16134](https://github.com/opensearch-project/OpenSearch/pull/16134)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle index 59063b3789c70..4d425964c77af 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle @@ -17,7 +17,7 @@ repositories { } dependencies { - implementation "org.apache.logging.log4j:log4j-core:2.24.0" + implementation "org.apache.logging.log4j:log4j-core:2.24.1" } ["0.0.1", "0.0.2"].forEach { v -> From 210228fcbe5c6d5efad417de64ae01c83927ed87 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 30 Sep 2024 21:11:50 -0400 Subject: [PATCH 080/108] Add 2.17.1 release notes (#16104) (#16105) (#16132) (cherry picked from commit dff2184279d7392f44cb52888916bdc9020a5e61) (cherry picked from commit af105b1bb547fc8843fab0986f1aa60454529d54) Signed-off-by: Andriy Redko --- release-notes/opensearch.release-notes-2.17.1.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 release-notes/opensearch.release-notes-2.17.1.md diff --git a/release-notes/opensearch.release-notes-2.17.1.md b/release-notes/opensearch.release-notes-2.17.1.md new file mode 100644 index 0000000000000..2ff2bd44c3b1c --- /dev/null +++ b/release-notes/opensearch.release-notes-2.17.1.md @@ -0,0 +1,16 @@ +## 2024-10-01 Version 2.17.1 Release Notes + +## [2.17.1] +### Added +- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664)) +- Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) + +### Dependencies + +### Changed + +### Deprecated + +### Removed + +### Fixed From 43e7597cdc5ba2c1852ec1796628f948633f0c57 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Tue, 1 Oct 2024 11:13:15 +0530 Subject: [PATCH 081/108] Add assertBusy to failing snapshot tests (#16146) Signed-off-by: Lakshya Taragi --- .../snapshots/SnapshotStatusApisIT.java | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java index c3214022df663..8b6869aa1d81a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java @@ -116,7 +116,7 @@ public void testStatusApiConsistency() { assertEquals(snapshotStatus.getStats().getTime(), snapshotInfo.endTime() - snapshotInfo.startTime()); } - public void testStatusAPICallForShallowCopySnapshot() { + public void testStatusAPICallForShallowCopySnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used for the test"); internalCluster().startClusterManagerOnlyNode(); internalCluster().startDataOnlyNode(); @@ -136,15 +136,24 @@ public void testStatusAPICallForShallowCopySnapshot() { final String snapshot = "snapshot"; createFullSnapshot(snapshotRepoName, snapshot); - final SnapshotStatus snapshotStatus = getSnapshotStatus(snapshotRepoName, snapshot); - assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); + assertBusy(() -> { + final SnapshotStatus snapshotStatus = client().admin() + .cluster() + .prepareSnapshotStatus(snapshotRepoName) + .setSnapshots(snapshot) + .execute() + .actionGet() + .getSnapshots() + .get(0); + assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); - final SnapshotIndexShardStatus snapshotShardState = stateFirstShard(snapshotStatus, indexName); - assertThat(snapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); - assertThat(snapshotShardState.getStats().getTotalFileCount(), greaterThan(0)); - assertThat(snapshotShardState.getStats().getTotalSize(), greaterThan(0L)); - assertThat(snapshotShardState.getStats().getIncrementalFileCount(), greaterThan(0)); - assertThat(snapshotShardState.getStats().getIncrementalSize(), greaterThan(0L)); + final SnapshotIndexShardStatus snapshotShardState = stateFirstShard(snapshotStatus, indexName); + assertThat(snapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); + assertThat(snapshotShardState.getStats().getTotalFileCount(), greaterThan(0)); + assertThat(snapshotShardState.getStats().getTotalSize(), greaterThan(0L)); + assertThat(snapshotShardState.getStats().getIncrementalFileCount(), greaterThan(0)); + assertThat(snapshotShardState.getStats().getIncrementalSize(), greaterThan(0L)); + }, 20, TimeUnit.SECONDS); } public void testStatusAPICallInProgressSnapshot() throws Exception { @@ -193,7 +202,7 @@ public void testExceptionOnMissingSnapBlob() throws IOException { ); } - public void testExceptionOnMissingShardLevelSnapBlob() throws IOException { + public void testExceptionOnMissingShardLevelSnapBlob() throws Exception { disableRepoConsistencyCheck("This test intentionally corrupts the repository"); final Path repoPath = randomRepoPath(); @@ -216,11 +225,12 @@ public void testExceptionOnMissingShardLevelSnapBlob() throws IOException { repoPath.resolve(resolvePath(indexId, "0")) .resolve(BlobStoreRepository.SNAPSHOT_PREFIX + snapshotInfo.snapshotId().getUUID() + ".dat") ); - - expectThrows( - SnapshotMissingException.class, - () -> client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap").execute().actionGet() - ); + assertBusy(() -> { + expectThrows( + SnapshotMissingException.class, + () -> client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap").execute().actionGet() + ); + }, 20, TimeUnit.SECONDS); } public void testGetSnapshotsWithoutIndices() throws Exception { From a767e92f3eeaf57c066deb9ad075d40ed00f4a58 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 1 Oct 2024 14:48:10 +0530 Subject: [PATCH 082/108] Optimize checksum creation for remote cluster state (#16046) * Support parallelisation in remote publication checksum computation Signed-off-by: Himshikha Gupta --- .../gateway/remote/ClusterStateChecksum.java | 150 ++++++++++++------ .../remote/RemoteClusterStateService.java | 14 +- .../org/opensearch/threadpool/ThreadPool.java | 7 + .../remote/ClusterMetadataManifestTests.java | 15 +- .../remote/ClusterStateChecksumTests.java | 32 ++-- .../RemoteClusterStateServiceTests.java | 28 ++-- 6 files changed, 164 insertions(+), 82 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index d6739c4572d1a..aa007f5da15b3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -12,8 +12,10 @@ import org.apache.logging.log4j.Logger; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.common.CheckedFunction; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -22,11 +24,15 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; import com.jcraft.jzlib.JZlib; @@ -37,6 +43,7 @@ */ public class ClusterStateChecksum implements ToXContentFragment, Writeable { + public static final int COMPONENT_SIZE = 11; static final String ROUTING_TABLE_CS = "routing_table"; static final String NODES_CS = "discovery_nodes"; static final String BLOCKS_CS = "blocks"; @@ -65,62 +72,103 @@ public class ClusterStateChecksum implements ToXContentFragment, Writeable { long indicesChecksum; long clusterStateChecksum; - public ClusterStateChecksum(ClusterState clusterState) { - try ( - BytesStreamOutput out = new BytesStreamOutput(); - BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out) - ) { - clusterState.routingTable().writeVerifiableTo(checksumOut); - routingTableChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - clusterState.nodes().writeVerifiableTo(checksumOut); - nodesChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - clusterState.coordinationMetadata().writeVerifiableTo(checksumOut); - coordinationMetadataChecksum = checksumOut.getChecksum(); - - // Settings create sortedMap by default, so no explicit sorting required here. - checksumOut.reset(); - Settings.writeSettingsToStream(clusterState.metadata().persistentSettings(), checksumOut); - settingMetadataChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - Settings.writeSettingsToStream(clusterState.metadata().transientSettings(), checksumOut); - transientSettingsMetadataChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - clusterState.metadata().templatesMetadata().writeVerifiableTo(checksumOut); - templatesMetadataChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - checksumOut.writeStringCollection(clusterState.metadata().customs().keySet()); - customMetadataMapChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeTo(checksumOut); - hashesOfConsistentSettingsChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - checksumOut.writeMapValues( + public ClusterStateChecksum(ClusterState clusterState, ThreadPool threadpool) { + long start = threadpool.relativeTimeInNanos(); + ExecutorService executorService = threadpool.executor(ThreadPool.Names.REMOTE_STATE_CHECKSUM); + CountDownLatch latch = new CountDownLatch(COMPONENT_SIZE); + + executeChecksumTask((stream) -> { + clusterState.routingTable().writeVerifiableTo(stream); + return null; + }, checksum -> routingTableChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + clusterState.nodes().writeVerifiableTo(stream); + return null; + }, checksum -> nodesChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + clusterState.coordinationMetadata().writeVerifiableTo(stream); + return null; + }, checksum -> coordinationMetadataChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + Settings.writeSettingsToStream(clusterState.metadata().persistentSettings(), stream); + return null; + }, checksum -> settingMetadataChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + Settings.writeSettingsToStream(clusterState.metadata().transientSettings(), stream); + return null; + }, checksum -> transientSettingsMetadataChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + clusterState.metadata().templatesMetadata().writeVerifiableTo(stream); + return null; + }, checksum -> templatesMetadataChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + stream.writeStringCollection(clusterState.metadata().customs().keySet()); + return null; + }, checksum -> customMetadataMapChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeTo(stream); + return null; + }, checksum -> hashesOfConsistentSettingsChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + stream.writeMapValues( clusterState.metadata().indices(), - (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream) + (checksumStream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) checksumStream) ); - indicesChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - clusterState.blocks().writeVerifiableTo(checksumOut); - blocksChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - checksumOut.writeStringCollection(clusterState.customs().keySet()); - clusterStateCustomsChecksum = checksumOut.getChecksum(); - } catch (IOException e) { - logger.error("Failed to create checksum for cluster state.", e); + return null; + }, checksum -> indicesChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + clusterState.blocks().writeVerifiableTo(stream); + return null; + }, checksum -> blocksChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + stream.writeStringCollection(clusterState.customs().keySet()); + return null; + }, checksum -> clusterStateCustomsChecksum = checksum, executorService, latch); + + try { + latch.await(); + } catch (InterruptedException e) { throw new RemoteStateTransferException("Failed to create checksum for cluster state.", e); } createClusterStateChecksum(); + logger.debug("Checksum execution time {}", TimeValue.nsecToMSec(threadpool.relativeTimeInNanos() - start)); + } + + private void executeChecksumTask( + CheckedFunction checksumTask, + Consumer checksumConsumer, + ExecutorService executorService, + CountDownLatch latch + ) { + executorService.execute(() -> { + try { + long checksum = createChecksum(checksumTask); + checksumConsumer.accept(checksum); + latch.countDown(); + } catch (IOException e) { + throw new RemoteStateTransferException("Failed to execute checksum task", e); + } + }); + } + + private long createChecksum(CheckedFunction task) throws IOException { + try ( + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out) + ) { + task.apply(checksumOut); + return checksumOut.getChecksum(); + } } private void createClusterStateChecksum() { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index ece29180f9cf5..ce5e57b79dadb 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -332,7 +332,9 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat uploadedMetadataResults, previousClusterUUID, clusterStateDiffManifest, - !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null, + !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) + ? new ClusterStateChecksum(clusterState, threadpool) + : null, false, codecVersion ); @@ -539,7 +541,9 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), clusterStateDiffManifest, - !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null, + !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) + ? new ClusterStateChecksum(clusterState, threadpool) + : null, false, previousManifest.getCodecVersion() ); @@ -1010,7 +1014,9 @@ public RemoteClusterStateManifestInfo markLastStateAsCommitted( uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), previousManifest.getDiffManifest(), - !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null, + !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) + ? new ClusterStateChecksum(clusterState, threadpool) + : null, true, previousManifest.getCodecVersion() ); @@ -1631,7 +1637,7 @@ void validateClusterStateFromChecksum( String localNodeId, boolean isFullStateDownload ) { - ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState); + ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState, threadpool); List failedValidation = newClusterStateChecksum.getMismatchEntities(manifest.getClusterStateChecksum()); if (failedValidation.isEmpty()) { return; diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 81220ab171b34..d795fd252b7fc 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -53,6 +53,7 @@ import org.opensearch.core.service.ReportingService; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.gateway.remote.ClusterStateChecksum; import org.opensearch.node.Node; import java.io.IOException; @@ -118,6 +119,7 @@ public static class Names { public static final String REMOTE_RECOVERY = "remote_recovery"; public static final String REMOTE_STATE_READ = "remote_state_read"; public static final String INDEX_SEARCHER = "index_searcher"; + public static final String REMOTE_STATE_CHECKSUM = "remote_state_checksum"; } /** @@ -191,6 +193,7 @@ public static ThreadPoolType fromType(String type) { map.put(Names.REMOTE_RECOVERY, ThreadPoolType.SCALING); map.put(Names.REMOTE_STATE_READ, ThreadPoolType.SCALING); map.put(Names.INDEX_SEARCHER, ThreadPoolType.RESIZABLE); + map.put(Names.REMOTE_STATE_CHECKSUM, ThreadPoolType.FIXED); THREAD_POOL_TYPES = Collections.unmodifiableMap(map); } @@ -307,6 +310,10 @@ public ThreadPool( runnableTaskListener ) ); + builders.put( + Names.REMOTE_STATE_CHECKSUM, + new FixedExecutorBuilder(settings, Names.REMOTE_STATE_CHECKSUM, ClusterStateChecksum.COMPONENT_SIZE, 1000) + ); for (final ExecutorBuilder builder : customBuilders) { if (builders.containsKey(builder.name())) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 3f9aa1245cab3..09c2933680be3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -34,6 +34,9 @@ import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.test.EqualsHashCodeTestUtils; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.junit.After; import java.io.IOException; import java.util.ArrayList; @@ -64,6 +67,14 @@ public class ClusterMetadataManifestTests extends OpenSearchTestCase { + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @After + public void teardown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + public void testClusterMetadataManifestXContentV0() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V0); ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() @@ -214,7 +225,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "indicesRoutingDiffPath" ) ) - .checksum(new ClusterStateChecksum(createClusterState())) + .checksum(new ClusterStateChecksum(createClusterState(), threadPool)) .build(); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( @@ -647,7 +658,7 @@ public void testClusterMetadataManifestXContentV4() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); final StringKeyDiffProvider routingTableIncrementalDiff = Mockito.mock(StringKeyDiffProvider.class); - ClusterStateChecksum checksum = new ClusterStateChecksum(createClusterState()); + ClusterStateChecksum checksum = new ClusterStateChecksum(createClusterState(), threadPool); ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() .clusterTerm(1L) .stateVersion(1L) diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java index 0203e56dd2d5c..9b98187053a39 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java @@ -34,6 +34,9 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.junit.After; import java.io.IOException; import java.util.EnumSet; @@ -41,14 +44,21 @@ import java.util.Map; public class ClusterStateChecksumTests extends OpenSearchTestCase { + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @After + public void teardown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } public void testClusterStateChecksumEmptyClusterState() { - ClusterStateChecksum checksum = new ClusterStateChecksum(ClusterState.EMPTY_STATE); + ClusterStateChecksum checksum = new ClusterStateChecksum(ClusterState.EMPTY_STATE, threadPool); assertNotNull(checksum); } public void testClusterStateChecksum() { - ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool); assertNotNull(checksum); assertTrue(checksum.routingTableChecksum != 0); assertTrue(checksum.nodesChecksum != 0); @@ -65,8 +75,8 @@ public void testClusterStateChecksum() { } public void testClusterStateMatchChecksum() { - ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); - ClusterStateChecksum newChecksum = new ClusterStateChecksum(generateClusterState()); + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool); + ClusterStateChecksum newChecksum = new ClusterStateChecksum(generateClusterState(), threadPool); assertNotNull(checksum); assertNotNull(newChecksum); assertEquals(checksum.routingTableChecksum, newChecksum.routingTableChecksum); @@ -84,7 +94,7 @@ public void testClusterStateMatchChecksum() { } public void testXContentConversion() throws IOException { - ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); checksum.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -97,7 +107,7 @@ public void testXContentConversion() throws IOException { } public void testSerialization() throws IOException { - ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool); BytesStreamOutput output = new BytesStreamOutput(); checksum.writeTo(output); @@ -109,10 +119,10 @@ public void testSerialization() throws IOException { public void testGetMismatchEntities() { ClusterState clsState1 = generateClusterState(); - ClusterStateChecksum checksum = new ClusterStateChecksum(clsState1); + ClusterStateChecksum checksum = new ClusterStateChecksum(clsState1, threadPool); assertTrue(checksum.getMismatchEntities(checksum).isEmpty()); - ClusterStateChecksum checksum2 = new ClusterStateChecksum(clsState1); + ClusterStateChecksum checksum2 = new ClusterStateChecksum(clsState1, threadPool); assertTrue(checksum.getMismatchEntities(checksum2).isEmpty()); ClusterState clsState2 = ClusterState.builder(ClusterName.DEFAULT) @@ -122,7 +132,7 @@ public void testGetMismatchEntities() { .customs(Map.of()) .metadata(Metadata.EMPTY_METADATA) .build(); - ClusterStateChecksum checksum3 = new ClusterStateChecksum(clsState2); + ClusterStateChecksum checksum3 = new ClusterStateChecksum(clsState2, threadPool); List mismatches = checksum.getMismatchEntities(checksum3); assertFalse(mismatches.isEmpty()); assertEquals(11, mismatches.size()); @@ -151,8 +161,8 @@ public void testGetMismatchEntitiesUnorderedInput() { ClusterState state2 = ClusterState.builder(state1).nodes(nodes1).build(); ClusterState state3 = ClusterState.builder(state1).nodes(nodes2).build(); - ClusterStateChecksum checksum1 = new ClusterStateChecksum(state2); - ClusterStateChecksum checksum2 = new ClusterStateChecksum(state3); + ClusterStateChecksum checksum1 = new ClusterStateChecksum(state2, threadPool); + ClusterStateChecksum checksum2 = new ClusterStateChecksum(state3, threadPool); assertEquals(checksum2, checksum1); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 56857285fa8d3..35a8ae16cacf7 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -3123,7 +3123,7 @@ public void testWriteFullMetadataSuccessWithChecksumValidationEnabled() throws I .previousClusterUUID("prev-cluster-uuid") .routingTableVersion(1L) .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) - .checksum(new ClusterStateChecksum(clusterState)) + .checksum(new ClusterStateChecksum(clusterState, threadPool)) .build(); assertThat(manifest.getIndices().size(), is(1)); @@ -3193,7 +3193,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationEnabled() t final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indices(Collections.emptyList()) - .checksum(new ClusterStateChecksum(clusterState)) + .checksum(new ClusterStateChecksum(clusterState, threadPool)) .build(); when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); @@ -3219,7 +3219,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationEnabled() t .previousClusterUUID("prev-cluster-uuid") .routingTableVersion(1) .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) - .checksum(new ClusterStateChecksum(clusterState)) + .checksum(new ClusterStateChecksum(clusterState, threadPool)) .build(); assertThat(manifest.getIndices().size(), is(1)); @@ -3245,7 +3245,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationModeNone() final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indices(Collections.emptyList()) - .checksum(new ClusterStateChecksum(clusterState)) + .checksum(new ClusterStateChecksum(clusterState, threadPool)) .build(); when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); @@ -3271,7 +3271,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationModeNone() .previousClusterUUID("prev-cluster-uuid") .routingTableVersion(1) .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) - .checksum(new ClusterStateChecksum(clusterState)) + .checksum(new ClusterStateChecksum(clusterState, threadPool)) .build(); assertThat(manifest.getIndices().size(), is(1)); @@ -3349,7 +3349,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).build(); remoteClusterStateService.start(); RemoteClusterStateService mockService = spy(remoteClusterStateService); @@ -3382,7 +3382,7 @@ public void testGetClusterStateForManifestWithChecksumValidationModeNone() throw initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.NONE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).build(); remoteClusterStateService.start(); RemoteClusterStateService mockService = spy(remoteClusterStateService); @@ -3415,7 +3415,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).build(); remoteClusterStateService.start(); RemoteClusterStateService mockService = spy(remoteClusterStateService); @@ -3465,7 +3465,7 @@ public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatc ); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).build(); remoteClusterStateService.start(); RemoteClusterStateService mockService = spy(remoteClusterStateService); @@ -3505,7 +3505,7 @@ public void testGetClusterStateUsingDiffWithChecksum() throws IOException { initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); remoteClusterStateService.start(); @@ -3547,7 +3547,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOExceptio initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.NONE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); remoteClusterStateService.start(); @@ -3589,7 +3589,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws I initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.DEBUG); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); remoteClusterStateService.start(); @@ -3630,7 +3630,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws I initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.TRACE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); remoteClusterStateService.start(); @@ -3692,7 +3692,7 @@ public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOExceptio initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); remoteClusterStateService.start(); From 1ef6444cfa1ab805ba5d68a3078274cbbb2b9a09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 09:14:28 -0400 Subject: [PATCH 083/108] Bump com.azure:azure-core-http-netty from 1.15.3 to 1.15.4 in /plugins/repository-azure (#16133) * Bump com.azure:azure-core-http-netty in /plugins/repository-azure Bumps [com.azure:azure-core-http-netty](https://github.com/Azure/azure-sdk-for-java) from 1.15.3 to 1.15.4. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-core-http-netty_1.15.3...azure-core-http-netty_1.15.4) --- updated-dependencies: - dependency-name: com.azure:azure-core-http-netty dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../licenses/azure-core-http-netty-1.15.3.jar.sha1 | 1 - .../licenses/azure-core-http-netty-1.15.4.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index da14948781849..038cc407d582b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) - Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) +- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 2892bdba51ba6..4baf79e619be9 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -48,7 +48,7 @@ dependencies { api 'com.azure:azure-json:1.1.0' api 'com.azure:azure-xml:1.1.0' api 'com.azure:azure-storage-common:12.25.1' - api 'com.azure:azure-core-http-netty:1.15.3' + api 'com.azure:azure-core-http-netty:1.15.4' api "io.netty:netty-codec-dns:${versions.netty}" api "io.netty:netty-codec-socks:${versions.netty}" api "io.netty:netty-codec-http2:${versions.netty}" diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1 deleted file mode 100644 index 3cea52ba67ce5..0000000000000 --- a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -03b5bd5f5c16eea71f130119dbfb1fe5239f806a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 new file mode 100644 index 0000000000000..97e6fad264294 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 @@ -0,0 +1 @@ +489a38c9e6efb5ce01fbd276d8cb6c0e89000459 \ No newline at end of file From be9f94258315405aee203a0737decd4e19343942 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Tue, 1 Oct 2024 19:07:29 +0530 Subject: [PATCH 084/108] [SnapshotV2] Add timestamp of last successful fetch of pinned timestamps in node stats (#15611) --------- Signed-off-by: Lakshya Taragi --- CHANGELOG.md | 1 + .../RemoteStorePinnedTimestampsIT.java | 41 ++++++++++ .../admin/cluster/node/stats/NodeStats.java | 26 +++++- .../cluster/node/stats/NodesStatsRequest.java | 3 +- .../node/stats/TransportNodesStatsAction.java | 3 +- .../stats/TransportClusterStatsAction.java | 1 + .../java/org/opensearch/node/NodeService.java | 7 +- .../remotestore/RemoteStoreNodeStats.java | 79 +++++++++++++++++++ .../cluster/node/stats/NodeStatsTests.java | 22 +++++- .../cluster/stats/ClusterStatsNodesTests.java | 1 + .../opensearch/cluster/DiskUsageTests.java | 6 ++ .../MockInternalClusterInfoService.java | 3 +- .../opensearch/test/InternalTestCluster.java | 1 + 13 files changed, 187 insertions(+), 7 deletions(-) create mode 100644 server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 038cc407d582b..1a0859a7d5af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) +- Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java index 2fcda8c2d2f27..024e0e952eea5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java @@ -9,6 +9,8 @@ package org.opensearch.remotestore; import org.opensearch.action.LatchedActionListener; +import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -20,6 +22,8 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; +import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.REMOTE_STORE; + @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStorePinnedTimestampsIT extends RemoteStoreBaseIntegTestCase { static final String INDEX_NAME = "remote-store-test-idx-1"; @@ -180,4 +184,41 @@ public void onFailure(Exception e) { assertBusy(() -> assertEquals(Set.of(timestamp2, timestamp3), RemoteStorePinnedTimestampService.getPinnedTimestamps().v2())); remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); } + + public void testLastSuccessfulFetchOfPinnedTimestampsPresentInNodeStats() throws Exception { + logger.info("Starting up cluster manager"); + logger.info("cluster.remote_store.pinned_timestamps.enabled set to true"); + logger.info("cluster.remote_store.pinned_timestamps.scheduler_interval set to minimum value of 1minute"); + Settings pinnedTimestampEnabledSettings = Settings.builder() + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_SCHEDULER_INTERVAL.getKey(), "1m") + .build(); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampEnabledSettings); + String remoteNodeName = internalCluster().startDataOnlyNodes(1, pinnedTimestampEnabledSettings).get(0); + ensureStableCluster(2); + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + remoteNodeName + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + assertBusy(() -> { + long lastSuccessfulFetchOfPinnedTimestamps = RemoteStorePinnedTimestampService.getPinnedTimestamps().v1(); + assertTrue(lastSuccessfulFetchOfPinnedTimestamps > 0L); + NodesStatsResponse nodesStatsResponse = internalCluster().client() + .admin() + .cluster() + .prepareNodesStats() + .addMetric(REMOTE_STORE.metricName()) + .execute() + .actionGet(); + for (NodeStats nodeStats : nodesStatsResponse.getNodes()) { + long lastRecordedFetch = nodeStats.getRemoteStoreNodeStats().getLastSuccessfulFetchOfPinnedTimestamps(); + assertTrue(lastRecordedFetch >= lastSuccessfulFetchOfPinnedTimestamps); + } + }); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index 0917a0baff1ab..c91260778f037 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -59,6 +59,7 @@ import org.opensearch.monitor.process.ProcessStats; import org.opensearch.node.AdaptiveSelectionStats; import org.opensearch.node.NodesResourceUsageStats; +import org.opensearch.node.remotestore.RemoteStoreNodeStats; import org.opensearch.ratelimitting.admissioncontrol.stats.AdmissionControlStats; import org.opensearch.repositories.RepositoriesStats; import org.opensearch.script.ScriptCacheStats; @@ -162,6 +163,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { @Nullable private NodeCacheStats nodeCacheStats; + @Nullable + private RemoteStoreNodeStats remoteStoreNodeStats; + public NodeStats(StreamInput in) throws IOException { super(in); timestamp = in.readVLong(); @@ -243,6 +247,12 @@ public NodeStats(StreamInput in) throws IOException { } else { nodeCacheStats = null; } + // TODO: change version to V_2_18_0 + if (in.getVersion().onOrAfter(Version.CURRENT)) { + remoteStoreNodeStats = in.readOptionalWriteable(RemoteStoreNodeStats::new); + } else { + remoteStoreNodeStats = null; + } } public NodeStats( @@ -274,7 +284,8 @@ public NodeStats( @Nullable SegmentReplicationRejectionStats segmentReplicationRejectionStats, @Nullable RepositoriesStats repositoriesStats, @Nullable AdmissionControlStats admissionControlStats, - @Nullable NodeCacheStats nodeCacheStats + @Nullable NodeCacheStats nodeCacheStats, + @Nullable RemoteStoreNodeStats remoteStoreNodeStats ) { super(node); this.timestamp = timestamp; @@ -305,6 +316,7 @@ public NodeStats( this.repositoriesStats = repositoriesStats; this.admissionControlStats = admissionControlStats; this.nodeCacheStats = nodeCacheStats; + this.remoteStoreNodeStats = remoteStoreNodeStats; } public long getTimestamp() { @@ -467,6 +479,11 @@ public NodeCacheStats getNodeCacheStats() { return nodeCacheStats; } + @Nullable + public RemoteStoreNodeStats getRemoteStoreNodeStats() { + return remoteStoreNodeStats; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -525,6 +542,10 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_14_0)) { out.writeOptionalWriteable(nodeCacheStats); } + // TODO: change version to V_2_18_0 + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeOptionalWriteable(remoteStoreNodeStats); + } } @Override @@ -631,6 +652,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (getNodeCacheStats() != null) { getNodeCacheStats().toXContent(builder, params); } + if (getRemoteStoreNodeStats() != null) { + getRemoteStoreNodeStats().toXContent(builder, params); + } return builder; } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java index f1f9f93afdad2..a5b00ed82d3cb 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -220,7 +220,8 @@ public enum Metric { SEGMENT_REPLICATION_BACKPRESSURE("segment_replication_backpressure"), REPOSITORIES("repositories"), ADMISSION_CONTROL("admission_control"), - CACHE_STATS("caches"); + CACHE_STATS("caches"), + REMOTE_STORE("remote_store"); private String metricName; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 2c808adc97c7a..a98d245af872b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -129,7 +129,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) { NodesStatsRequest.Metric.SEGMENT_REPLICATION_BACKPRESSURE.containedIn(metrics), NodesStatsRequest.Metric.REPOSITORIES.containedIn(metrics), NodesStatsRequest.Metric.ADMISSION_CONTROL.containedIn(metrics), - NodesStatsRequest.Metric.CACHE_STATS.containedIn(metrics) + NodesStatsRequest.Metric.CACHE_STATS.containedIn(metrics), + NodesStatsRequest.Metric.REMOTE_STORE.containedIn(metrics) ); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index a49ca2035783c..c4b3524cf6da5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -174,6 +174,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false, false, false, + false, false ); List shardsStats = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java index 1eb38ea63ad5a..9671fda14375d 100644 --- a/server/src/main/java/org/opensearch/node/NodeService.java +++ b/server/src/main/java/org/opensearch/node/NodeService.java @@ -54,6 +54,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.ingest.IngestService; import org.opensearch.monitor.MonitorService; +import org.opensearch.node.remotestore.RemoteStoreNodeStats; import org.opensearch.plugins.PluginsService; import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlService; import org.opensearch.repositories.RepositoriesService; @@ -241,7 +242,8 @@ public NodeStats stats( boolean segmentReplicationTrackerStats, boolean repositoriesStats, boolean admissionControl, - boolean cacheService + boolean cacheService, + boolean remoteStoreNodeStats ) { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) @@ -274,7 +276,8 @@ public NodeStats stats( segmentReplicationTrackerStats ? this.segmentReplicationStatsTracker.getTotalRejectionStats() : null, repositoriesStats ? this.repositoriesService.getRepositoriesStats() : null, admissionControl ? this.admissionControlService.stats() : null, - cacheService ? this.cacheService.stats(indices) : null + cacheService ? this.cacheService.stats(indices) : null, + remoteStoreNodeStats ? new RemoteStoreNodeStats() : null ); } diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java new file mode 100644 index 0000000000000..8da8a17e21839 --- /dev/null +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.node.remotestore; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * Node level remote store stats + * @opensearch.internal + */ +public class RemoteStoreNodeStats implements Writeable, ToXContentFragment { + + public static final String STATS_NAME = "remote_store"; + public static final String LAST_SUCCESSFUL_FETCH_OF_PINNED_TIMESTAMPS = "last_successful_fetch_of_pinned_timestamps"; + + /** + * Time stamp for the last successful fetch of pinned timestamps by the {@linkplain RemoteStorePinnedTimestampService} + */ + private final long lastSuccessfulFetchOfPinnedTimestamps; + + public RemoteStoreNodeStats() { + this.lastSuccessfulFetchOfPinnedTimestamps = RemoteStorePinnedTimestampService.getPinnedTimestamps().v1(); + } + + public long getLastSuccessfulFetchOfPinnedTimestamps() { + return this.lastSuccessfulFetchOfPinnedTimestamps; + } + + public RemoteStoreNodeStats(StreamInput in) throws IOException { + this.lastSuccessfulFetchOfPinnedTimestamps = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(this.lastSuccessfulFetchOfPinnedTimestamps); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(STATS_NAME); + builder.field(LAST_SUCCESSFUL_FETCH_OF_PINNED_TIMESTAMPS, this.lastSuccessfulFetchOfPinnedTimestamps); + return builder.endObject(); + } + + @Override + public String toString() { + return "RemoteStoreNodeStats{ lastSuccessfulFetchOfPinnedTimestamps=" + lastSuccessfulFetchOfPinnedTimestamps + "}"; + } + + @Override + public boolean equals(Object o) { + if (o == null) { + return false; + } + if (o.getClass() != RemoteStoreNodeStats.class) { + return false; + } + RemoteStoreNodeStats other = (RemoteStoreNodeStats) o; + return this.lastSuccessfulFetchOfPinnedTimestamps == other.lastSuccessfulFetchOfPinnedTimestamps; + } + + @Override + public int hashCode() { + return Objects.hash(lastSuccessfulFetchOfPinnedTimestamps); + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index 11902728eed07..34065daff2b8a 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -95,6 +95,7 @@ import org.opensearch.node.NodeResourceUsageStats; import org.opensearch.node.NodesResourceUsageStats; import org.opensearch.node.ResponseCollectorService; +import org.opensearch.node.remotestore.RemoteStoreNodeStats; import org.opensearch.ratelimitting.admissioncontrol.controllers.AdmissionController; import org.opensearch.ratelimitting.admissioncontrol.controllers.CpuBasedAdmissionController; import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType; @@ -614,6 +615,14 @@ public void testSerialization() throws IOException { } else { assertEquals(nodeCacheStats, deserializedNodeCacheStats); } + + RemoteStoreNodeStats remoteStoreNodeStats = nodeStats.getRemoteStoreNodeStats(); + RemoteStoreNodeStats deserializedRemoteStoreNodeStats = deserializedNodeStats.getRemoteStoreNodeStats(); + if (remoteStoreNodeStats == null) { + assertNull(deserializedRemoteStoreNodeStats); + } else { + assertEquals(remoteStoreNodeStats, deserializedRemoteStoreNodeStats); + } } } } @@ -996,6 +1005,16 @@ public void apply(String action, AdmissionControlActionType admissionControlActi nodeCacheStats = new NodeCacheStats(cacheStatsMap, flags); } + RemoteStoreNodeStats remoteStoreNodeStats = null; + if (frequently()) { + remoteStoreNodeStats = new RemoteStoreNodeStats() { + @Override + public long getLastSuccessfulFetchOfPinnedTimestamps() { + return 123456L; + } + }; + } + // TODO: Only remote_store based aspects of NodeIndicesStats are being tested here. // It is possible to test other metrics in NodeIndicesStats as well since it extends Writeable now return new NodeStats( @@ -1027,7 +1046,8 @@ public void apply(String action, AdmissionControlActionType admissionControlActi segmentReplicationRejectionStats, null, admissionControlStats, - nodeCacheStats + nodeCacheStats, + remoteStoreNodeStats ); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java index 1c4a77905d73f..823661ba14abf 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java @@ -349,6 +349,7 @@ private ClusterStatsNodeResponse createClusterStatsNodeResponse( null, null, null, + null, null ); if (defaultBehavior) { diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java index 5539dd26dd52d..cd050fb346563 100644 --- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java +++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java @@ -195,6 +195,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ), new NodeStats( @@ -226,6 +227,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ), new NodeStats( @@ -257,6 +259,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ) ); @@ -319,6 +322,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ), new NodeStats( @@ -350,6 +354,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ), new NodeStats( @@ -381,6 +386,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ) ); diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java index 35ca5d80aeb4e..ded457601c0ae 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java @@ -125,7 +125,8 @@ List adjustNodesStats(List nodesStats) { nodeStats.getSegmentReplicationRejectionStats(), nodeStats.getRepositoriesStats(), nodeStats.getAdmissionControlStats(), - nodeStats.getNodeCacheStats() + nodeStats.getNodeCacheStats(), + nodeStats.getRemoteStoreNodeStats() ); }).collect(Collectors.toList()); } diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 7adff82e72245..fa5fb736f518f 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -2752,6 +2752,7 @@ public void ensureEstimatedStats() { false, false, false, + false, false ); assertThat( From c46bf412fd046b9b3f7bbd6e8ba10e2fc57dcfa3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:33:35 -0400 Subject: [PATCH 085/108] Bump org.jline:jline from 3.26.3 to 3.27.0 in /test/fixtures/hdfs-fixture (#16135) * Bump org.jline:jline in /test/fixtures/hdfs-fixture Bumps [org.jline:jline](https://github.com/jline/jline3) from 3.26.3 to 3.27.0. - [Release notes](https://github.com/jline/jline3/releases) - [Changelog](https://github.com/jline/jline3/blob/master/changelog.md) - [Commits](https://github.com/jline/jline3/compare/jline-parent-3.26.3...jline-3.27.0) --- updated-dependencies: - dependency-name: org.jline:jline dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a0859a7d5af8..3df8aec0f376a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) +- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 99b0386a48808..bc63362980bea 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -77,7 +77,7 @@ dependencies { api "ch.qos.logback:logback-core:1.5.8" api "ch.qos.logback:logback-classic:1.2.13" api "org.jboss.xnio:xnio-nio:3.8.16.Final" - api 'org.jline:jline:3.26.3' + api 'org.jline:jline:3.27.0' api 'org.apache.commons:commons-configuration2:2.11.0' api 'com.nimbusds:nimbus-jose-jwt:9.41.1' api ('org.apache.kerby:kerb-admin:2.1.0') { From 7b248efc5ab50573ed25a9eaac5d44c9b75c243a Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 1 Oct 2024 12:09:02 -0400 Subject: [PATCH 086/108] Remove Identity FeatureFlag (#16024) * Remove Identity FeatureFlag Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Mark IdentityService as internal Signed-off-by: Craig Perkins * Add @InternalApi annotation Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 3 ++- .../org/opensearch/http/IdentityAuthenticationIT.java | 1 - .../opensearch/common/settings/FeatureFlagSettings.java | 1 - .../java/org/opensearch/common/util/FeatureFlags.java | 8 -------- .../java/org/opensearch/identity/IdentityService.java | 4 +++- server/src/main/java/org/opensearch/node/Node.java | 6 +----- .../org/opensearch/common/util/FeatureFlagTests.java | 9 +++------ 7 files changed, 9 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3df8aec0f376a..ef56029b31eae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) -- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) @@ -37,6 +36,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) +- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) +- Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024)) ### Deprecated diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java index 1a806b033eb8a..14346b8910c76 100644 --- a/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java +++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java @@ -38,7 +38,6 @@ public class IdentityAuthenticationIT extends HttpSmokeTestCase { protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(FeatureFlags.IDENTITY, "true") .build(); } diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 73375f9eaa413..59d999798868e 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -31,7 +31,6 @@ protected FeatureFlagSettings( public static final Set> BUILT_IN_FEATURE_FLAGS = Set.of( FeatureFlags.EXTENSIONS_SETTING, - FeatureFlags.IDENTITY_SETTING, FeatureFlags.TELEMETRY_SETTING, FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.TIERED_REMOTE_INDEX_SETTING, diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index a5acea004c3b2..6df68013a8119 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -40,11 +40,6 @@ public class FeatureFlags { */ public static final String EXTENSIONS = "opensearch.experimental.feature.extensions.enabled"; - /** - * Gates the functionality of identity. - */ - public static final String IDENTITY = "opensearch.experimental.feature.identity.enabled"; - /** * Gates the functionality of telemetry framework. */ @@ -82,8 +77,6 @@ public class FeatureFlags { public static final Setting EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope); - public static final Setting IDENTITY_SETTING = Setting.boolSetting(IDENTITY, false, Property.NodeScope); - public static final Setting TELEMETRY_SETTING = Setting.boolSetting(TELEMETRY, false, Property.NodeScope); public static final Setting DATETIME_FORMATTER_CACHING_SETTING = Setting.boolSetting( @@ -138,7 +131,6 @@ public class FeatureFlags { private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, - IDENTITY_SETTING, TELEMETRY_SETTING, DATETIME_FORMATTER_CACHING_SETTING, TIERED_REMOTE_INDEX_SETTING, diff --git a/server/src/main/java/org/opensearch/identity/IdentityService.java b/server/src/main/java/org/opensearch/identity/IdentityService.java index 03f937180f4ba..33066fae5a80d 100644 --- a/server/src/main/java/org/opensearch/identity/IdentityService.java +++ b/server/src/main/java/org/opensearch/identity/IdentityService.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; +import org.opensearch.common.annotation.InternalApi; import org.opensearch.common.settings.Settings; import org.opensearch.identity.noop.NoopIdentityPlugin; import org.opensearch.identity.tokens.TokenManager; @@ -22,8 +23,9 @@ /** * Identity and access control for OpenSearch * - * @opensearch.experimental + * @opensearch.internal * */ +@InternalApi public class IdentityService { private static final Logger log = LogManager.getLogger(IdentityService.class); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 4962d72d8728a..56d04df5921ee 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -526,11 +526,7 @@ protected Node( FeatureFlags.initializeFeatureFlags(settings); final List identityPlugins = new ArrayList<>(); - if (FeatureFlags.isEnabled(FeatureFlags.IDENTITY)) { - // If identity is enabled load plugins implementing the extension point - logger.info("Identity on so found plugins implementing: " + pluginsService.filterPlugins(IdentityPlugin.class).toString()); - identityPlugins.addAll(pluginsService.filterPlugins(IdentityPlugin.class)); - } + identityPlugins.addAll(pluginsService.filterPlugins(IdentityPlugin.class)); final Set additionalRoles = pluginsService.filterPlugins(Plugin.class) .stream() diff --git a/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java b/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java index 6b6c7b96266d3..6d9d1aad3c5d5 100644 --- a/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java +++ b/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java @@ -14,7 +14,6 @@ import static org.opensearch.common.util.FeatureFlags.DATETIME_FORMATTER_CACHING; import static org.opensearch.common.util.FeatureFlags.EXTENSIONS; -import static org.opensearch.common.util.FeatureFlags.IDENTITY; public class FeatureFlagTests extends OpenSearchTestCase { @@ -40,7 +39,7 @@ public void testNonBooleanFeatureFlag() { } public void testBooleanFeatureFlagWithDefaultSetToFalse() { - final String testFlag = IDENTITY; + final String testFlag = EXTENSIONS; FeatureFlags.initializeFeatureFlags(Settings.EMPTY); assertNotNull(testFlag); assertFalse(FeatureFlags.isEnabled(testFlag)); @@ -49,15 +48,13 @@ public void testBooleanFeatureFlagWithDefaultSetToFalse() { public void testBooleanFeatureFlagInitializedWithEmptySettingsAndDefaultSetToFalse() { final String testFlag = DATETIME_FORMATTER_CACHING; FeatureFlags.initializeFeatureFlags(Settings.EMPTY); - assertNotNull(testFlag); assertFalse(FeatureFlags.isEnabled(testFlag)); } public void testInitializeFeatureFlagsWithExperimentalSettings() { - FeatureFlags.initializeFeatureFlags(Settings.builder().put(IDENTITY, true).build()); - assertTrue(FeatureFlags.isEnabled(IDENTITY)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(EXTENSIONS, true).build()); + assertTrue(FeatureFlags.isEnabled(EXTENSIONS)); assertFalse(FeatureFlags.isEnabled(DATETIME_FORMATTER_CACHING)); - assertFalse(FeatureFlags.isEnabled(EXTENSIONS)); // reset FeatureFlags to defaults FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } From 908fefeed32ce205e7b6128a5a344bbf333a5e62 Mon Sep 17 00:00:00 2001 From: Arpit-Bandejiya Date: Tue, 1 Oct 2024 22:01:59 +0530 Subject: [PATCH 087/108] Change default retry mechanism of s3 client to Standard Mode (#15978) Signed-off-by: Arpit Bandejiya --- CHANGELOG.md | 1 + .../java/org/opensearch/repositories/s3/S3AsyncService.java | 5 ++++- .../main/java/org/opensearch/repositories/s3/S3Service.java | 3 +++ .../opensearch/repositories/s3/AwsS3ServiceImplTests.java | 3 ++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef56029b31eae..14232ad87478e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) - Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) +- [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java index d691cad9c9d03..8bbef168de89c 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java @@ -18,6 +18,7 @@ import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.http.async.SdkAsyncHttpClient; @@ -239,7 +240,9 @@ static ClientOverrideConfiguration buildOverrideConfiguration(final S3ClientSett RetryPolicy.builder() .numRetries(clientSettings.maxRetries) .throttlingBackoffStrategy( - clientSettings.throttleRetries ? BackoffStrategy.defaultThrottlingStrategy() : BackoffStrategy.none() + clientSettings.throttleRetries + ? BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD) + : BackoffStrategy.none() ) .build() ) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java index fe81da31432f4..3d5e121778ba9 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java @@ -42,6 +42,7 @@ import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.http.SystemPropertyTlsKeyManagersProvider; @@ -330,6 +331,8 @@ static ClientOverrideConfiguration buildOverrideConfiguration(final S3ClientSett ); if (!clientSettings.throttleRetries) { retryPolicy.throttlingBackoffStrategy(BackoffStrategy.none()); + } else { + retryPolicy.throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD)); } return clientOverrideConfiguration.retryPolicy(retryPolicy.build()).build(); } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java index b80b857644f2a..e7312157d7a33 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java @@ -35,6 +35,7 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.http.apache.ProxyConfiguration; @@ -364,7 +365,7 @@ private void launchAWSConfigurationTest( if (expectedUseThrottleRetries) { assertThat( clientOverrideConfiguration.retryPolicy().get().throttlingBackoffStrategy(), - is(BackoffStrategy.defaultThrottlingStrategy()) + is(BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD)) ); } else { assertThat(clientOverrideConfiguration.retryPolicy().get().throttlingBackoffStrategy(), is(BackoffStrategy.none())); From 0b1650de442d7c84f532bd23ae8e766abf1d5c45 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 1 Oct 2024 16:40:38 -0400 Subject: [PATCH 088/108] Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler (#16154) * Ensure RestHandler.Wrapper delegates all implementations to the wrapper handler Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Fix typo Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + .../java/org/opensearch/rest/RestHandler.java | 10 +++++ .../opensearch/rest/BaseRestHandlerTests.java | 41 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14232ad87478e..2fb5fd6f2757e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024)) +- Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/rest/RestHandler.java b/server/src/main/java/org/opensearch/rest/RestHandler.java index 143cbd472ed07..7c3a369be61b9 100644 --- a/server/src/main/java/org/opensearch/rest/RestHandler.java +++ b/server/src/main/java/org/opensearch/rest/RestHandler.java @@ -192,6 +192,16 @@ public List replacedRoutes() { public boolean allowSystemIndexAccessByDefault() { return delegate.allowSystemIndexAccessByDefault(); } + + @Override + public boolean isActionPaginated() { + return delegate.isActionPaginated(); + } + + @Override + public boolean supportsStreaming() { + return delegate.supportsStreaming(); + } } /** diff --git a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java index 45653e9d8e4d6..7534dcd93944a 100644 --- a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java +++ b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java @@ -35,6 +35,8 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.common.Table; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestHandler.ReplacedRoute; import org.opensearch.rest.RestHandler.Route; import org.opensearch.rest.RestRequest.Method; @@ -46,15 +48,22 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.lang.reflect.Modifier; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import static org.hamcrest.core.StringContains.containsString; import static org.hamcrest.object.HasToString.hasToString; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; public class BaseRestHandlerTests extends OpenSearchTestCase { private NodeClient mockClient; @@ -288,4 +297,36 @@ public void testReplaceRoutesMethod() throws Exception { } } + public void testRestHandlerWrapper() throws Exception { + RestHandler rh = new RestHandler() { + @Override + public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { + new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY); + } + }; + RestHandler handlerSpy = spy(rh); + RestHandler.Wrapper rhWrapper = new RestHandler.Wrapper(handlerSpy); + + List overridableMethods = Arrays.stream(RestHandler.class.getMethods()) + .filter( + m -> !(Modifier.isPrivate(m.getModifiers()) || Modifier.isStatic(m.getModifiers()) || Modifier.isFinal(m.getModifiers())) + ) + .collect(Collectors.toList()); + + for (java.lang.reflect.Method method : overridableMethods) { + int argCount = method.getParameterCount(); + Object[] args = new Object[argCount]; + for (int i = 0; i < argCount; i++) { + args[i] = any(); + } + if (args.length > 0) { + method.invoke(rhWrapper, args); + } else { + method.invoke(rhWrapper); + } + method.invoke(verify(handlerSpy, times(1)), args); + } + verifyNoMoreInteractions(handlerSpy); + } + } From 1ee858fc8cc631d35c4a48b2a53d4ab6009e2c2b Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 1 Oct 2024 19:33:06 -0400 Subject: [PATCH 089/108] [BUG] Streaming bulk request hangs (#16158) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + .../rest/ReactorNetty4StreamingIT.java | 28 +++++++++++++++++++ .../org/opensearch/rest/RestController.java | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fb5fd6f2757e..333b1cd19b404 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) - Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521)) +- Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158)) ### Security diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java index 6f3895fffa437..1b60023da0329 100644 --- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java +++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.Locale; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -297,4 +298,31 @@ public void testStreamingBadStream() throws IOException { assertThat(streamingResponse.getStatusLine().getStatusCode(), equalTo(200)); assertThat(streamingResponse.getWarnings(), empty()); } + + public void testStreamingLargeDocument() throws IOException { + final Stream stream = Stream.of( + String.format( + Locale.getDefault(), + "{ \"index\": { \"_index\": \"test-streaming\", \"_id\": \"1\" } }\n{ \"name\": \"%s\" }\n", + randomAlphaOfLength(5000) + ) + ); + + final Duration delay = Duration.ofMillis(1); + final StreamingRequest streamingRequest = new StreamingRequest<>( + "POST", + "/_bulk/stream", + Flux.fromStream(stream).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8))) + ); + + final StreamingResponse streamingResponse = client().streamRequest(streamingRequest); + + StepVerifier.create(Flux.from(streamingResponse.getBody()).map(b -> new String(b.array(), StandardCharsets.UTF_8))) + .expectNextMatches(s -> s.contains("\"type\":\"illegal_argument_exception\"")) + .expectComplete() + .verify(); + + assertThat(streamingResponse.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(streamingResponse.getWarnings(), empty()); + } } diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java index 4f87c01258396..c17f723c13f2a 100644 --- a/server/src/main/java/org/opensearch/rest/RestController.java +++ b/server/src/main/java/org/opensearch/rest/RestController.java @@ -709,7 +709,7 @@ public void sendResponse(RestResponse response) { prepareResponse(response.status(), Map.of("Content-Type", List.of(response.contentType()))); } - Mono.ignoreElements(this).then(Mono.just(response)).subscribe(delegate::sendResponse); + Mono.from(this).ignoreElement().then(Mono.just(response)).subscribe(delegate::sendResponse); } @Override From 79a2ec1daa06296ebfe6ca23e6fc64f720d2d9f4 Mon Sep 17 00:00:00 2001 From: prachi-gaonkar Date: Wed, 2 Oct 2024 18:51:51 +0530 Subject: [PATCH 090/108] Fix client-rest-high-level tests for ppc64le (#16089) * Fix client-rest-high-level tests for ppc64le Signed-off-by: prachi-gaonkar * Fix client-rest-high-level tests for ppc64le Signed-off-by: prachi-gaonkar --------- Signed-off-by: prachi-gaonkar --- .../client/OpenSearchRestHighLevelClientTestCase.java | 1 + .../src/test/java/org/opensearch/client/RankEvalIT.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java index b0a7d1e3578c0..b512117c42f65 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java @@ -90,6 +90,7 @@ public abstract class OpenSearchRestHighLevelClientTestCase extends OpenSearchRestTestCase { protected static final String CONFLICT_PIPELINE_ID = "conflict_pipeline"; + protected static final double DOUBLE_DELTA = 0.000001; private static RestHighLevelClient restHighLevelClient; private static boolean async = Booleans.parseBoolean(System.getProperty("tests.rest.async", "false")); diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java index 01fdd489aa7d8..6da8a29a9789e 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java @@ -158,7 +158,7 @@ public void testMetrics() throws IOException { RankEvalRequest rankEvalRequest = new RankEvalRequest(spec, new String[] { "index", "index2" }); RankEvalResponse response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); - assertEquals(expectedScores[i], response.getMetricScore(), Double.MIN_VALUE); + assertEquals(expectedScores[i], response.getMetricScore(), DOUBLE_DELTA); i++; } } From 4e3a6d038dfc37660c7b9e47f570a32b0f2be4d4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:56:13 -0400 Subject: [PATCH 091/108] [AUTO] [main] Add bwc version 2.17.2. (#16166) * Add bwc version 2.17.2 Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update Version.java Signed-off-by: Andriy Redko --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Andriy Redko Co-authored-by: opensearch-ci-bot <83309141+opensearch-ci-bot@users.noreply.github.com> Co-authored-by: Andriy Redko --- .ci/bwcVersions | 1 + README.md | 2 +- libs/core/src/main/java/org/opensearch/Version.java | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index ccb47c1a3b724..5587b8e5784c5 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -39,4 +39,5 @@ BWC_VERSION: - "2.16.1" - "2.17.0" - "2.17.1" + - "2.17.2" - "2.18.0" diff --git a/README.md b/README.md index 03728aee0135c..95fbac7bbecf1 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) [![2.18.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0") -[![2.17.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.1") +[![2.17.2 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.2)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.2") [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index b86b8459fb8a8..5d38f85c40c89 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -110,6 +110,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_16_1 = new Version(2160199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1); + public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version CURRENT = V_3_0_0; From 23fcfba74ab701ff9b544f441eb54d59bf61e036 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Wed, 2 Oct 2024 16:41:28 -0400 Subject: [PATCH 092/108] Fixes license Signed-off-by: Darshit Chanpura --- .../resources/ResourceService.java | 5 +++- .../accesscontrol/resources/package-info.java | 27 ------------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java index 4c403fed834ff..1a5fd9c8d8701 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java @@ -1,6 +1,9 @@ /* - * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ package org.opensearch.accesscontrol.resources; diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java b/server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java index 65fe5c5f05713..b8ad5237a60ca 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java @@ -6,33 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * Actions that OpenSearch can take either on the data stored on disk or on other nodes. - */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - /** * This package defines all classes required for Resource Sharing and Access Control */ From fba48abd7e0353e96b9b4f2247618d978bf3c37f Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Wed, 2 Oct 2024 16:44:14 -0400 Subject: [PATCH 093/108] Adds changelog entry Signed-off-by: Darshit Chanpura --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 333b1cd19b404..cf4c76e0276f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) - Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) - [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) +- Add resource-level access control and sharing ([#16030](https://github.com/opensearch-project/OpenSearch/pull/16030)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) From 9cb8d0e9b653f077101e9e0a9e61b5db5c329d64 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Wed, 2 Oct 2024 17:11:26 -0400 Subject: [PATCH 094/108] Adds a notion of scope Signed-off-by: Darshit Chanpura --- .../resources/ResourceAccessScope.java | 21 +++++++++++++++++++ .../NoOpResourceAccessControlPlugin.java | 3 ++- .../plugins/ResourceAccessControlPlugin.java | 3 ++- 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 server/src/main/java/org/opensearch/accesscontrol/resources/ResourceAccessScope.java diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceAccessScope.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceAccessScope.java new file mode 100644 index 0000000000000..463936625522c --- /dev/null +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceAccessScope.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.accesscontrol.resources; + +/** + * This interface defines the two basic access scopes for resource-access. + * Each plugin must implement their own scopes and manage them + * These access scopes will then be used to verify the type of access being requested. + * + * @opensearch.experimental + */ +public interface ResourceAccessScope { + String READ_ONLY = "read_only"; + String READ_WRITE = "read_write"; +} diff --git a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java index c3fae97380186..2619b05124b67 100644 --- a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java @@ -46,10 +46,11 @@ public List listAccessibleResourcesForPlugin(String systemIndexName) { /** * @param resourceId the resource on which access is to be checked * @param systemIndexName where the resource exists + * @param scope the type of access being requested * @return true */ @Override - public boolean hasPermission(String resourceId, String systemIndexName) { + public boolean hasPermission(String resourceId, String systemIndexName, String scope) { return true; } diff --git a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java index f23fdefda0230..8298a65becfd7 100644 --- a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java @@ -45,9 +45,10 @@ public interface ResourceAccessControlPlugin { * * @param resourceId the resource on which access is to be checked * @param systemIndexName where the resource exists + * @param scope the scope being requested * @return true if current user has access, false otherwise */ - boolean hasPermission(String resourceId, String systemIndexName); + boolean hasPermission(String resourceId, String systemIndexName, String scope); /** * Adds an entity to the share-with. Resource needs to be in restricted mode. From e7ad37db8037409a2ca4e6660a7faa0651a60c7e Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:01:14 +0530 Subject: [PATCH 095/108] Change version checks from CURRENT to 2.18 (#16174) Signed-off-by: Lakshya Taragi --- .../action/admin/cluster/node/stats/NodeStats.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index c91260778f037..eb79e3403a25c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -247,8 +247,7 @@ public NodeStats(StreamInput in) throws IOException { } else { nodeCacheStats = null; } - // TODO: change version to V_2_18_0 - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { remoteStoreNodeStats = in.readOptionalWriteable(RemoteStoreNodeStats::new); } else { remoteStoreNodeStats = null; @@ -542,8 +541,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_14_0)) { out.writeOptionalWriteable(nodeCacheStats); } - // TODO: change version to V_2_18_0 - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeOptionalWriteable(remoteStoreNodeStats); } } From bf6566e76c46bd7dc5d838503b8c43ff8cb09757 Mon Sep 17 00:00:00 2001 From: Sumit Bansal Date: Thu, 3 Oct 2024 13:29:26 +0530 Subject: [PATCH 096/108] Add changes to block calls in cat shards, indices and segments based on dynamic limit settings (#15986) * Add changes to block calls in cat shards, indices and segments based on dynamic limit settings Signed-off-by: Sumit Bansal --- CHANGELOG.md | 1 + .../opensearch/OpenSearchServerException.java | 9 + .../org/opensearch/action/ActionModule.java | 11 +- .../cluster/shards/CatShardsRequest.java | 10 + .../shards/TransportCatShardsAction.java | 31 ++- .../ResponseLimitBreachedException.java | 66 +++++ .../common/breaker/ResponseLimitSettings.java | 180 ++++++++++++++ .../common/settings/ClusterSettings.java | 8 +- .../rest/action/cat/AbstractCatAction.java | 9 + .../rest/action/cat/RestIndicesAction.java | 25 +- .../rest/action/cat/RestSegmentsAction.java | 25 ++ .../rest/action/cat/RestShardsAction.java | 6 + .../action/list/RestIndicesListAction.java | 10 + .../ExceptionSerializationTests.java | 2 + .../RenamedTimeoutRequestParameterTests.java | 12 +- .../breaker/ResponseLimitSettingsTests.java | 228 ++++++++++++++++++ .../action/cat/RestIndicesActionTests.java | 14 +- 17 files changed, 636 insertions(+), 11 deletions(-) create mode 100644 server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java create mode 100644 server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java create mode 100644 server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 333b1cd19b404..b69c33df3b6a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) - Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) - [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) +- Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index b0ab62259d5c0..0ee3debb4819a 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -14,6 +14,7 @@ import static org.opensearch.Version.V_2_10_0; import static org.opensearch.Version.V_2_13_0; import static org.opensearch.Version.V_2_17_0; +import static org.opensearch.Version.V_2_18_0; import static org.opensearch.Version.V_2_1_0; import static org.opensearch.Version.V_2_4_0; import static org.opensearch.Version.V_2_5_0; @@ -1210,6 +1211,14 @@ public static void registerExceptions() { V_2_17_0 ) ); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.common.breaker.ResponseLimitBreachedException.class, + org.opensearch.common.breaker.ResponseLimitBreachedException::new, + 175, + V_2_18_0 + ) + ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.cluster.block.IndexCreateBlockException.class, diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 937d7509fe68c..98bcd6ba9c3f8 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -300,6 +300,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.NamedRegistry; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.inject.AbstractModule; import org.opensearch.common.inject.TypeLiteral; import org.opensearch.common.inject.multibindings.MapBinder; @@ -531,6 +532,7 @@ public class ActionModule extends AbstractModule { private final RequestValidators indicesAliasesRequestRequestValidators; private final ThreadPool threadPool; private final ExtensionsManager extensionsManager; + private final ResponseLimitSettings responseLimitSettings; public ActionModule( Settings settings, @@ -583,6 +585,7 @@ public ActionModule( ); restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService); + responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); } public Map> getActions() { @@ -968,8 +971,8 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestClusterManagerAction()); registerHandler.accept(new RestNodesAction()); registerHandler.accept(new RestTasksAction(nodesInCluster)); - registerHandler.accept(new RestIndicesAction()); - registerHandler.accept(new RestSegmentsAction()); + registerHandler.accept(new RestIndicesAction(responseLimitSettings)); + registerHandler.accept(new RestSegmentsAction(responseLimitSettings)); // Fully qualified to prevent interference with rest.action.count.RestCountAction registerHandler.accept(new org.opensearch.rest.action.cat.RestCountAction()); // Fully qualified to prevent interference with rest.action.indices.RestRecoveryAction @@ -989,7 +992,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestTemplatesAction()); // LIST API - registerHandler.accept(new RestIndicesListAction()); + registerHandler.accept(new RestIndicesListAction(responseLimitSettings)); // Point in time API registerHandler.accept(new RestCreatePitAction()); @@ -1060,6 +1063,8 @@ protected void configure() { // register dynamic ActionType -> transportAction Map used by NodeClient bind(DynamicActionRegistry.class).toInstance(dynamicActionRegistry); + + bind(ResponseLimitSettings.class).toInstance(responseLimitSettings); } public ActionFilters getActionFilters() { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java index 49299777db8ae..76aa25b9a96b5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java @@ -27,11 +27,13 @@ public class CatShardsRequest extends ClusterManagerNodeReadRequest headers) { return new ClusterAdminTask(id, type, action, parentTaskId, headers, this.cancelAfterTimeInterval); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java index 224d3cbc5f10a..6b073a16f7d28 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java @@ -16,6 +16,8 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.TimeoutTaskCancellationUtility; import org.opensearch.client.node.NodeClient; +import org.opensearch.common.breaker.ResponseLimitBreachedException; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.NotifyOnceListener; @@ -23,6 +25,10 @@ import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; +import java.util.Objects; + +import static org.opensearch.common.breaker.ResponseLimitSettings.LimitEntity.SHARDS; + /** * Perform cat shards action * @@ -31,11 +37,18 @@ public class TransportCatShardsAction extends HandledTransportAction { private final NodeClient client; + private final ResponseLimitSettings responseLimitSettings; @Inject - public TransportCatShardsAction(NodeClient client, TransportService transportService, ActionFilters actionFilters) { + public TransportCatShardsAction( + NodeClient client, + TransportService transportService, + ActionFilters actionFilters, + ResponseLimitSettings responseLimitSettings + ) { super(CatShardsAction.NAME, transportService, actionFilters, CatShardsRequest::new); this.client = client; + this.responseLimitSettings = responseLimitSettings; } @Override @@ -73,6 +86,7 @@ protected void innerOnFailure(Exception e) { client.admin().cluster().state(clusterStateRequest, new ActionListener() { @Override public void onResponse(ClusterStateResponse clusterStateResponse) { + validateRequestLimit(shardsRequest, clusterStateResponse, cancellableListener); catShardsResponse.setClusterStateResponse(clusterStateResponse); IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); indicesStatsRequest.setShouldCancelOnTimeout(true); @@ -107,4 +121,19 @@ public void onFailure(Exception e) { } } + + private void validateRequestLimit( + final CatShardsRequest shardsRequest, + final ClusterStateResponse clusterStateResponse, + final ActionListener listener + ) { + if (shardsRequest.isRequestLimitCheckSupported() + && Objects.nonNull(clusterStateResponse) + && Objects.nonNull(clusterStateResponse.getState())) { + int limit = responseLimitSettings.getCatShardsResponseLimit(); + if (ResponseLimitSettings.isResponseLimitBreached(clusterStateResponse.getState().getRoutingTable(), SHARDS, limit)) { + listener.onFailure(new ResponseLimitBreachedException("Too many shards requested.", limit, SHARDS)); + } + } + } } diff --git a/server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java new file mode 100644 index 0000000000000..db6785067edf6 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.breaker; + +import org.opensearch.OpenSearchException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Thrown when api response breaches threshold limit. + * + * @opensearch.internal + */ +public class ResponseLimitBreachedException extends OpenSearchException { + + private final int responseLimit; + private final ResponseLimitSettings.LimitEntity limitEntity; + + public ResponseLimitBreachedException(StreamInput in) throws IOException { + super(in); + responseLimit = in.readVInt(); + limitEntity = in.readEnum(ResponseLimitSettings.LimitEntity.class); + } + + public ResponseLimitBreachedException(String msg, int responseLimit, ResponseLimitSettings.LimitEntity limitEntity) { + super(msg); + this.responseLimit = responseLimit; + this.limitEntity = limitEntity; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVInt(responseLimit); + out.writeEnum(limitEntity); + } + + public int getResponseLimit() { + return responseLimit; + } + + public ResponseLimitSettings.LimitEntity getLimitEntity() { + return limitEntity; + } + + @Override + public RestStatus status() { + return RestStatus.TOO_MANY_REQUESTS; + } + + @Override + protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("response_limit", responseLimit); + builder.field("limit_entity", limitEntity); + } +} diff --git a/server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java new file mode 100644 index 0000000000000..8eb47c9aaf147 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java @@ -0,0 +1,180 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.breaker; + +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.rest.action.cat.RestIndicesAction; +import org.opensearch.rest.action.cat.RestSegmentsAction; +import org.opensearch.rest.action.cat.RestShardsAction; + +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + +/** + * Class to define dynamic settings for putting response limits on the actions and methods to evaluate if block is required. + */ +public class ResponseLimitSettings { + + /** + * Enum to represent entity against which we need to perform limit checks. + */ + public enum LimitEntity { + INDICES, + SHARDS + } + + private volatile int catIndicesResponseLimit; + private volatile int catShardsResponseLimit; + private volatile int catSegmentsResponseLimit; + + /** + * Setting to enable response limit on {@link RestIndicesAction}. The limit will be applied on number of indices. + */ + public static final Setting CAT_INDICES_RESPONSE_LIMIT_SETTING = Setting.intSetting( + "cat.indices.response.limit.number_of_indices", + -1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Setting to enable response limit on {@link RestShardsAction}. The limit will be applied on number of shards. + */ + public static final Setting CAT_SHARDS_RESPONSE_LIMIT_SETTING = Setting.intSetting( + "cat.shards.response.limit.number_of_shards", + -1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Setting to enable response limit on {@link RestSegmentsAction}. The limit will be applied on number of indices. + */ + public static final Setting CAT_SEGMENTS_RESPONSE_LIMIT_SETTING = Setting.intSetting( + "cat.segments.response.limit.number_of_indices", + -1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public ResponseLimitSettings(ClusterSettings clusterSettings, Settings settings) { + setCatShardsResponseLimit(CAT_SHARDS_RESPONSE_LIMIT_SETTING.get(settings)); + setCatIndicesResponseLimit(CAT_INDICES_RESPONSE_LIMIT_SETTING.get(settings)); + setCatSegmentsResponseLimit(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.get(settings)); + + clusterSettings.addSettingsUpdateConsumer(CAT_SHARDS_RESPONSE_LIMIT_SETTING, this::setCatShardsResponseLimit); + clusterSettings.addSettingsUpdateConsumer(CAT_INDICES_RESPONSE_LIMIT_SETTING, this::setCatIndicesResponseLimit); + clusterSettings.addSettingsUpdateConsumer(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING, this::setCatSegmentsResponseLimit); + } + + /** + * Method to check if the response limit has reached for an action. + * The limits are controlled via dynamic settings. + * + * @param metadata {@link Metadata} + * @param limitEntity {@link LimitEntity} + * @param limit Integer limit on block entity + * @return True/False + */ + public static boolean isResponseLimitBreached(final Metadata metadata, final LimitEntity limitEntity, final int limit) { + if (Objects.isNull(metadata) || limit <= 0) return false; + if (limitEntity == LimitEntity.INDICES) { + int indicesCount = getTotalIndicesFromMetadata.apply(metadata); + return indicesCount > limit; + } else { + throw new IllegalArgumentException("Unsupported limit entity [" + limitEntity + "]"); + } + } + + /** + * Method to check if the response limit has reached for an action. + * The limits are controlled via dynamic settings. + * + * @param routingTable {@link RoutingTable} + * @param limitEntity {@link LimitEntity} + * @param limit Integer limit on block entity + * @return True/False + */ + public static boolean isResponseLimitBreached(final RoutingTable routingTable, final LimitEntity limitEntity, final int limit) { + if (Objects.isNull(routingTable) || limit <= 0) return false; + if (Objects.isNull(limitEntity)) { + throw new IllegalArgumentException("Limit entity cannot be null"); + } + switch (limitEntity) { + case INDICES: + int indicesCount = getTotalIndicesFromRoutingTable.apply(routingTable); + if (indicesCount > limit) return true; + break; + case SHARDS: + if (isShardsLimitBreached(routingTable, limit)) return true; + break; + default: + throw new IllegalArgumentException("Unsupported limit entity [" + limitEntity + "]"); + } + return false; + } + + private static boolean isShardsLimitBreached(final RoutingTable routingTable, final int limit) { + final Map indexRoutingTableMap = routingTable.getIndicesRouting(); + int totalShards = 0; + for (final Map.Entry entry : indexRoutingTableMap.entrySet()) { + for (final Map.Entry indexShardRoutingTableEntry : entry.getValue().getShards().entrySet()) { + totalShards += indexShardRoutingTableEntry.getValue().getShards().size(); + // Fail fast if limit value is breached and avoid unnecessary computation. + if (totalShards > limit) return true; + } + } + return false; + } + + private void setCatShardsResponseLimit(final int catShardsResponseLimit) { + this.catShardsResponseLimit = catShardsResponseLimit; + } + + private void setCatIndicesResponseLimit(final int catIndicesResponseLimit) { + this.catIndicesResponseLimit = catIndicesResponseLimit; + } + + private void setCatSegmentsResponseLimit(final int catSegmentsResponseLimit) { + this.catSegmentsResponseLimit = catSegmentsResponseLimit; + } + + public int getCatShardsResponseLimit() { + return this.catShardsResponseLimit; + } + + public int getCatIndicesResponseLimit() { + return this.catIndicesResponseLimit; + } + + public int getCatSegmentsResponseLimit() { + return this.catSegmentsResponseLimit; + } + + static Function getTotalIndicesFromMetadata = (metadata) -> { + if (Objects.nonNull(metadata) && Objects.nonNull(metadata.getIndices())) { + return metadata.getIndices().size(); + } + return 0; + }; + + static Function getTotalIndicesFromRoutingTable = (routingTable) -> { + if (Objects.nonNull(routingTable) && Objects.nonNull(routingTable.getIndicesRouting())) { + return routingTable.getIndicesRouting().size(); + } + return 0; + }; +} diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index ecdd23530c648..e18c4dabb29eb 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -84,6 +84,7 @@ import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.cache.CacheType; import org.opensearch.common.cache.settings.CacheSettings; import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings; @@ -796,7 +797,12 @@ public void apply(Settings value, Settings current, Settings previous) { WorkloadManagementSettings.NODE_LEVEL_CPU_REJECTION_THRESHOLD, WorkloadManagementSettings.NODE_LEVEL_CPU_CANCELLATION_THRESHOLD, WorkloadManagementSettings.NODE_LEVEL_MEMORY_REJECTION_THRESHOLD, - WorkloadManagementSettings.NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD + WorkloadManagementSettings.NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD, + + // Settings to be used for limiting rest requests + ResponseLimitSettings.CAT_INDICES_RESPONSE_LIMIT_SETTING, + ResponseLimitSettings.CAT_SHARDS_RESPONSE_LIMIT_SETTING, + ResponseLimitSettings.CAT_SEGMENTS_RESPONSE_LIMIT_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java index 6f4e060363bfb..506f5f1529776 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java @@ -33,6 +33,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.common.Table; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.io.Streams; import org.opensearch.common.io.UTF8StreamWriter; import org.opensearch.core.common.io.stream.BytesStream; @@ -98,4 +99,12 @@ protected Set responseParams() { return RESPONSE_PARAMS; } + /** + * Method to check if limits defined in {@link ResponseLimitSettings} are applicable to an action. + * + * @return True / False status + */ + protected boolean isRequestLimitCheckSupported() { + return false; + } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 1e76008ff8c64..7562e09ddd704 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -50,6 +50,8 @@ import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Table; +import org.opensearch.common.breaker.ResponseLimitBreachedException; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.collect.Tuple; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; @@ -86,6 +88,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest.DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT; +import static org.opensearch.common.breaker.ResponseLimitSettings.LimitEntity.INDICES; import static org.opensearch.rest.RestRequest.Method.GET; /** @@ -102,6 +105,12 @@ public class RestIndicesAction extends AbstractListAction { private static final String DUPLICATE_PARAMETER_ERROR_MESSAGE = "Please only use one of the request parameters [master_timeout, cluster_manager_timeout]."; + private final ResponseLimitSettings responseLimitSettings; + + public RestIndicesAction(ResponseLimitSettings responseLimitSettings) { + this.responseLimitSettings = responseLimitSettings; + } + @Override public List routes() { return unmodifiableList(asList(new Route(GET, "/_cat/indices"), new Route(GET, "/_cat/indices/{index}"))); @@ -123,6 +132,11 @@ protected void documentation(StringBuilder sb) { sb.append("/_cat/indices/{index}\n"); } + @Override + public boolean isRequestLimitCheckSupported() { + return true; + } + @Override public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); @@ -162,7 +176,6 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) { // type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so // force the IndicesOptions for all the sub-requests to be as inclusive as possible. final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden(); - // Indices that were successfully resolved during the get settings request might be deleted when the // subsequent cluster state, cluster health and indices stats requests execute. We have to distinguish two cases: // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we @@ -181,6 +194,7 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) { new ActionListener() { @Override public void onResponse(ClusterStateResponse clusterStateResponse) { + validateRequestLimit(clusterStateResponse, listener); IndexPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); // For non-paginated queries, indicesToBeQueried would be same as indices retrieved from // rest request and unresolved, while for paginated queries, it would be a list of indices @@ -234,6 +248,15 @@ public void onFailure(final Exception e) { } + private void validateRequestLimit(final ClusterStateResponse clusterStateResponse, final ActionListener

listener) { + if (isRequestLimitCheckSupported() && Objects.nonNull(clusterStateResponse) && Objects.nonNull(clusterStateResponse.getState())) { + int limit = responseLimitSettings.getCatIndicesResponseLimit(); + if (ResponseLimitSettings.isResponseLimitBreached(clusterStateResponse.getState().getMetadata(), INDICES, limit)) { + listener.onFailure(new ResponseLimitBreachedException("Too many indices requested.", limit, INDICES)); + } + } + } + /** * We're using the Get Settings API here to resolve the authorized indices for the user. * This is because the Cluster State and Cluster Health APIs do not filter output based diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java index b88af4ac3eeed..69216cb46492d 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java @@ -42,6 +42,8 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.Table; +import org.opensearch.common.breaker.ResponseLimitBreachedException; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.core.common.Strings; import org.opensearch.index.engine.Segment; @@ -52,9 +54,11 @@ import java.util.List; import java.util.Map; +import java.util.Objects; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; +import static org.opensearch.common.breaker.ResponseLimitSettings.LimitEntity.INDICES; import static org.opensearch.rest.RestRequest.Method.GET; /** @@ -66,6 +70,12 @@ public class RestSegmentsAction extends AbstractCatAction { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestSegmentsAction.class); + private final ResponseLimitSettings responseLimitSettings; + + public RestSegmentsAction(ResponseLimitSettings responseLimitSettings) { + this.responseLimitSettings = responseLimitSettings; + } + @Override public List routes() { return unmodifiableList(asList(new Route(GET, "/_cat/segments"), new Route(GET, "/_cat/segments/{index}"))); @@ -81,6 +91,11 @@ public boolean allowSystemIndexAccessByDefault() { return true; } + @Override + public boolean isRequestLimitCheckSupported() { + return true; + } + @Override public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); @@ -96,6 +111,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener(channel) { @Override public void processResponse(final ClusterStateResponse clusterStateResponse) { + validateRequestLimit(clusterStateResponse); final IndicesSegmentsRequest indicesSegmentsRequest = new IndicesSegmentsRequest(); indicesSegmentsRequest.indices(indices); client.admin().indices().segments(indicesSegmentsRequest, new RestResponseListener(channel) { @@ -110,6 +126,15 @@ public RestResponse buildResponse(final IndicesSegmentResponse indicesSegmentRes }); } + private void validateRequestLimit(final ClusterStateResponse clusterStateResponse) { + if (isRequestLimitCheckSupported() && Objects.nonNull(clusterStateResponse) && Objects.nonNull(clusterStateResponse.getState())) { + int limit = responseLimitSettings.getCatSegmentsResponseLimit(); + if (ResponseLimitSettings.isResponseLimitBreached(clusterStateResponse.getState().getRoutingTable(), INDICES, limit)) { + throw new ResponseLimitBreachedException("Segments from too many indices requested.", limit, INDICES); + } + } + } + @Override protected void documentation(StringBuilder sb) { sb.append("/_cat/segments\n"); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java index a7ad5fe6c14a3..c8b4e7472927e 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java @@ -105,6 +105,11 @@ protected void documentation(StringBuilder sb) { sb.append("/_cat/shards/{index}\n"); } + @Override + public boolean isRequestLimitCheckSupported() { + return true; + } + @Override public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); @@ -113,6 +118,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli shardsRequest.clusterManagerNodeTimeout(request.paramAsTime("cluster_manager_timeout", shardsRequest.clusterManagerNodeTimeout())); shardsRequest.setCancelAfterTimeInterval(request.paramAsTime("cancel_after_time_interval", NO_TIMEOUT)); shardsRequest.setIndices(indices); + shardsRequest.setRequestLimitCheckSupported(isRequestLimitCheckSupported()); parseDeprecatedMasterTimeoutParameter(shardsRequest, request, deprecationLogger, getName()); return channel -> client.execute(CatShardsAction.INSTANCE, shardsRequest, new RestResponseListener(channel) { @Override diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index ad5c58c86ce90..971cab0535be9 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -9,6 +9,7 @@ package org.opensearch.rest.action.list; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.rest.RestRequest; @@ -35,6 +36,10 @@ public class RestIndicesListAction extends RestIndicesAction { private static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE = 5000; private static final int DEFAULT_LIST_INDICES_PAGE_SIZE = 500; + public RestIndicesListAction(final ResponseLimitSettings responseLimitSettings) { + super(responseLimitSettings); + } + @Override public List routes() { return unmodifiableList(asList(new Route(GET, "/_list/indices"), new Route(GET, "/_list/indices/{index}"))); @@ -70,6 +75,11 @@ protected PageParams validateAndGetPageParams(RestRequest restRequest) { return pageParams; } + @Override + public boolean isRequestLimitCheckSupported() { + return false; + } + protected int defaultPageSize() { return DEFAULT_LIST_INDICES_PAGE_SIZE; } diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index 2e4a2d7bdd59c..c1972daeab6d3 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -65,6 +65,7 @@ import org.opensearch.cluster.routing.UnsupportedWeightedRoutingStateException; import org.opensearch.cluster.service.ClusterManagerThrottlingException; import org.opensearch.common.UUIDs; +import org.opensearch.common.breaker.ResponseLimitBreachedException; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -898,6 +899,7 @@ public void testIds() { ids.put(172, ViewNotFoundException.class); ids.put(173, ViewAlreadyExistsException.class); ids.put(174, InvalidIndexContextException.class); + ids.put(175, ResponseLimitBreachedException.class); ids.put(10001, IndexCreateBlockException.class); Map, Integer> reverse = new HashMap<>(); diff --git a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java index c7e1039686cc9..26c4670b4288c 100644 --- a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java +++ b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java @@ -11,7 +11,9 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.client.node.NodeClient; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.logging.DeprecationLogger; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; import org.opensearch.core.common.bytes.BytesArray; @@ -155,7 +157,10 @@ public void testCatAllocation() { } public void testCatIndices() { - RestIndicesAction action = new RestIndicesAction(); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + Settings settings = Settings.builder().build(); + ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + RestIndicesAction action = new RestIndicesAction(responseLimitSettings); Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(getRestRequestWithBothParams(), client)); assertThat(e.getMessage(), containsString(DUPLICATE_PARAMETER_ERROR_MESSAGE)); assertWarnings(MASTER_TIMEOUT_DEPRECATED_MESSAGE); @@ -239,7 +244,10 @@ public void testCatThreadPool() { } public void testCatSegments() { - RestSegmentsAction action = new RestSegmentsAction(); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final Settings settings = Settings.builder().build(); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + RestSegmentsAction action = new RestSegmentsAction(responseLimitSettings); Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(getRestRequestWithBothParams(), client)); assertThat(e.getMessage(), containsString(DUPLICATE_PARAMETER_ERROR_MESSAGE)); assertWarnings(MASTER_TIMEOUT_DEPRECATED_MESSAGE); diff --git a/server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java b/server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java new file mode 100644 index 0000000000000..4633d055b3505 --- /dev/null +++ b/server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java @@ -0,0 +1,228 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.breaker; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.TestShardRouting; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.common.breaker.ResponseLimitSettings.CAT_INDICES_RESPONSE_LIMIT_SETTING; +import static org.opensearch.common.breaker.ResponseLimitSettings.CAT_SEGMENTS_RESPONSE_LIMIT_SETTING; +import static org.opensearch.common.breaker.ResponseLimitSettings.CAT_SHARDS_RESPONSE_LIMIT_SETTING; + +public class ResponseLimitSettingsTests extends OpenSearchTestCase { + + public void testIsResponseLimitBreachedForNullMetadataExpectNotBreached() { + final Settings settings = Settings.builder().build(); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + final boolean breached = ResponseLimitSettings.isResponseLimitBreached( + (Metadata) null, + ResponseLimitSettings.LimitEntity.INDICES, + 0 + ); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForNullRoutingTableExpectNotBreached() { + final Settings settings = Settings.builder().build(); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + final boolean breached = ResponseLimitSettings.isResponseLimitBreached( + (RoutingTable) null, + ResponseLimitSettings.LimitEntity.INDICES, + 0 + ); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForNullLimitEntityWithRoutingTableExpectException() { + final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3"); + expectThrows( + IllegalArgumentException.class, + () -> ResponseLimitSettings.isResponseLimitBreached(clusterState.getRoutingTable(), null, 4) + ); + } + + public void testIsResponseLimitBreachedForNullLimitEntityWithMetadataExpectException() { + final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3"); + expectThrows( + IllegalArgumentException.class, + () -> ResponseLimitSettings.isResponseLimitBreached(clusterState.getMetadata(), null, 4) + ); + } + + public void testIsResponseLimitBreachedForCatIndicesWithNoSettingPassedExpectNotBreached() { + // Don't set setting + final Settings settings = Settings.builder().build(); + final boolean breached = isBreachedForIndices(settings); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForCatIndicesWithSettingDisabledExpectNotBreached() { + // Don't enable limit + final Settings settings = Settings.builder().put(CAT_INDICES_RESPONSE_LIMIT_SETTING.getKey(), -1).build(); + final boolean breached = isBreachedForIndices(settings); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForCatIndicesWithSettingEnabledExpectBreached() { + // Set limit of 1 index + final Settings settings = Settings.builder().put(CAT_INDICES_RESPONSE_LIMIT_SETTING.getKey(), 1).build(); + final boolean breached = isBreachedForIndices(settings); + assertTrue(breached); + } + + public void testIsResponseLimitBreachedForCatIndicesWithSettingEnabledExpectNotBreached() { + // Set limit of 5 indices + final Settings settings = Settings.builder().put(CAT_INDICES_RESPONSE_LIMIT_SETTING.getKey(), 5).build(); + final boolean breached = isBreachedForIndices(settings); + assertFalse(breached); + } + + private static boolean isBreachedForIndices(final Settings settings) { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + // Pass cluster state with 3 indices + final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3"); + return ResponseLimitSettings.isResponseLimitBreached( + clusterState.getMetadata(), + ResponseLimitSettings.LimitEntity.INDICES, + responseLimitSettings.getCatIndicesResponseLimit() + ); + } + + public void testIsResponseLimitBreachedForCatShardsWithSettingDisabledExpectNotBreached() { + // Don't enable limit + final Settings settings = Settings.builder().put(CAT_SHARDS_RESPONSE_LIMIT_SETTING.getKey(), -1).build(); + final boolean breached = isBreachedForShards(settings); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForCatShardsWithNoSettingPassedExpectNotBreached() { + // Don't set setting + final Settings settings = Settings.builder().build(); + final boolean breached = isBreachedForShards(settings); + assertFalse(breached); + } + + private static boolean isBreachedForShards(Settings settings) { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + // Build cluster state with 3 shards + final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3"); + return ResponseLimitSettings.isResponseLimitBreached( + clusterState.getRoutingTable(), + ResponseLimitSettings.LimitEntity.SHARDS, + responseLimitSettings.getCatShardsResponseLimit() + ); + } + + public void testIsResponseLimitBreachedForCatShardsWithSettingEnabledExpectBreached() { + // Set limit of 2 shards + final Settings settings = Settings.builder().put(CAT_SHARDS_RESPONSE_LIMIT_SETTING.getKey(), 2).build(); + final boolean breached = isBreachedForShards(settings); + assertTrue(breached); + } + + public void testIsResponseLimitBreachedForCatShardsWithSettingEnabledExpectNotBreached() { + // Set limit of 9 shards + final Settings settings = Settings.builder().put(CAT_SHARDS_RESPONSE_LIMIT_SETTING.getKey(), 9).build(); + final boolean breached = isBreachedForShards(settings); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForCatSegmentsWithSettingDisabledExpectNotBreached() { + // Don't enable limit + final Settings settings = Settings.builder().put(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.getKey(), -1).build(); + final boolean breached = isBreachedForSegments(settings); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForCatSegmentsWithNoSettingPassedExpectNotBreached() { + // Don't set setting + final Settings settings = Settings.builder().build(); + final boolean breached = isBreachedForSegments(settings); + assertFalse(breached); + } + + private static boolean isBreachedForSegments(Settings settings) { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + // Build cluster state with 3 indices + final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3"); + final boolean breached = ResponseLimitSettings.isResponseLimitBreached( + clusterState.getRoutingTable(), + ResponseLimitSettings.LimitEntity.INDICES, + responseLimitSettings.getCatSegmentsResponseLimit() + ); + return breached; + } + + public void testIsResponseLimitBreachedForCatSegmentsWithSettingEnabledExpectBreached() { + // Set limit of 1 index + final Settings settings = Settings.builder().put(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.getKey(), 1).build(); + final boolean breached = isBreachedForSegments(settings); + assertTrue(breached); + } + + public void testIsResponseLimitBreachedForCatSegmentsWithSettingEnabledExpectNotBreached() { + // Set limit of 5 indices + final Settings settings = Settings.builder().put(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.getKey(), 5).build(); + final boolean breached = isBreachedForSegments(settings); + assertFalse(breached); + } + + private static ClusterState buildClusterState(String... indices) { + final Metadata.Builder metadata = Metadata.builder(); + for (String index : indices) { + metadata.put(IndexMetadata.builder(index).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)); + } + final Map indexRoutingTableMap = new HashMap<>(); + for (String s : indices) { + final Index index = new Index(s, "uuid"); + final ShardId primaryShardId = new ShardId(index, 0); + final ShardRouting primaryShardRouting = createShardRouting(primaryShardId, true); + final ShardId replicaShardId = new ShardId(index, 1); + final ShardRouting replicaShardRouting = createShardRouting(replicaShardId, false); + final IndexShardRoutingTable.Builder indexShardRoutingTableBuilder = new IndexShardRoutingTable.Builder(primaryShardId); + indexShardRoutingTableBuilder.addShard(primaryShardRouting); + indexShardRoutingTableBuilder.addShard(replicaShardRouting); + final IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index) + .addShard(primaryShardRouting) + .addShard(replicaShardRouting) + .addIndexShard(indexShardRoutingTableBuilder.build()); + indexRoutingTableMap.put(index.getName(), indexRoutingTable.build()); + } + final RoutingTable routingTable = new RoutingTable(1, indexRoutingTableMap); + return ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(routingTable) + .build(); + } + + private static ShardRouting createShardRouting(ShardId shardId, boolean isPrimary) { + return TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(4), isPrimary, ShardRoutingState.STARTED); + } +} diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 1d1b509ae94e5..2debb00a0d14f 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -43,6 +43,8 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.Table; import org.opensearch.common.UUIDs; +import org.opensearch.common.breaker.ResponseLimitSettings; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; @@ -144,7 +146,10 @@ public void setup() { } public void testBuildTable() { - final RestIndicesAction action = new RestIndicesAction(); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final Settings settings = Settings.builder().build(); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + final RestIndicesAction action = new RestIndicesAction(responseLimitSettings); final Table table = action.buildTable( new FakeRestRequest(), indicesSettings, @@ -165,8 +170,11 @@ public void testBuildTable() { } public void testBuildPaginatedTable() { - final RestIndicesAction action = new RestIndicesAction(); - final RestIndicesListAction indicesListAction = new RestIndicesListAction(); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final Settings settings = Settings.builder().build(); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + final RestIndicesAction action = new RestIndicesAction(responseLimitSettings); + final RestIndicesListAction indicesListAction = new RestIndicesListAction(responseLimitSettings); List indicesList = new ArrayList<>(indicesMetadatas.keySet()); // Using half of the indices from metadata list for a page String[] indicesToBeQueried = indicesList.subList(0, indicesMetadatas.size() / 2).toArray(new String[0]); From 5771e811e1a2d8339d897a9ba08689ad3f3efb4a Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 3 Oct 2024 17:10:57 +0530 Subject: [PATCH 097/108] Separate Remote State and Publication enabled and configured methods (#16080) * Separate Remote State and Publication enabled and configured methods Signed-off-by: Shivansh Arora --- .../RemotePublicationConfigurationIT.java | 26 ++++++++++++++ .../remote/RemoteStatePublicationIT.java | 6 +++- .../coordination/CoordinationState.java | 2 +- .../cluster/coordination/Coordinator.java | 8 +++-- .../coordination/JoinTaskExecutor.java | 4 +-- .../PublicationTransportHandler.java | 14 ++------ .../cluster/node/DiscoveryNode.java | 4 +-- .../remote/RemoteClusterStateService.java | 6 ++-- .../index/remote/RemoteIndexPathUploader.java | 4 +-- .../main/java/org/opensearch/node/Node.java | 4 +-- .../remotestore/RemoteStoreNodeAttribute.java | 5 ++- .../test/OpenSearchIntegTestCase.java | 35 +++++++++++-------- 12 files changed, 74 insertions(+), 44 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java index 1d19a4bfd1af5..98859f517aad4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java @@ -18,6 +18,7 @@ import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; +import org.junit.Assert; import org.junit.Before; import java.util.Collection; @@ -118,6 +119,31 @@ public Settings.Builder remoteWithRoutingTableNodeSetting() { .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true); } + public void testRemoteClusterStateServiceNotInitialized_WhenNodeAttributesNotPresent() { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + ensureStableCluster(3); + ensureGreen(); + + internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNull); + } + + public void testServiceInitialized_WhenNodeAttributesPresent() { + internalCluster().startClusterManagerOnlyNode( + buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) + ); + internalCluster().startDataOnlyNodes( + 2, + buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) + ); + + ensureStableCluster(3); + ensureGreen(); + + internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNotNull); + } + public void testRemotePublishConfigNodeJoinNonRemoteCluster() throws Exception { internalCluster().startClusterManagerOnlyNode(); internalCluster().startDataOnlyNodes(2); diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index 578c922c80a0d..2409b5d0d0e45 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -215,7 +215,11 @@ public void testRemotePublicationDisableIfRemoteStateDisabled() { ensureStableCluster(5); ensureGreen(INDEX_NAME); - assertNull(internalCluster().getCurrentClusterManagerNodeInstance(RemoteClusterStateService.class)); + RemoteClusterStateService remoteClusterStateService = internalCluster().getCurrentClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + + assertFalse(remoteClusterStateService.isRemotePublicationEnabled()); } public void testRemotePublicationDownloadStats() { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index 9cffc7051d756..e5b07f867d609 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -108,7 +108,7 @@ public CoordinationState( // ToDo: revisit this check while making the setting dynamic this.isRemotePublicationEnabled = isRemoteStateEnabled && REMOTE_PUBLICATION_SETTING.get(settings) - && localNode.isRemoteStatePublicationEnabled(); + && localNode.isRemoteStatePublicationConfigured(); } public boolean isRemotePublicationEnabled() { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 9859abe503eaa..1b3ae89251ac0 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -189,6 +189,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private final PersistedStateRegistry persistedStateRegistry; private final RemoteStoreNodeService remoteStoreNodeService; private NodeConnectionsService nodeConnectionsService; + private final RemoteClusterStateService remoteClusterStateService; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -312,6 +313,7 @@ public Coordinator( this.persistedStateRegistry = persistedStateRegistry; this.localNodeCommissioned = true; this.remoteStoreNodeService = remoteStoreNodeService; + this.remoteClusterStateService = remoteClusterStateService; } private ClusterFormationState getClusterFormationState() { @@ -912,9 +914,9 @@ public DiscoveryStats stats() { stats.add(persistedStateRegistry.getPersistedState(stateType).getStats()); } }); - if (coordinationState.get().isRemotePublicationEnabled()) { - stats.add(publicationHandler.getFullDownloadStats()); - stats.add(publicationHandler.getDiffDownloadStats()); + if (remoteClusterStateService != null) { + stats.add(remoteClusterStateService.getFullDownloadStats()); + stats.add(remoteClusterStateService.getDiffDownloadStats()); } clusterStateStats.setPersistenceStats(stats); return new DiscoveryStats(new PendingClusterStateStats(0, 0, 0), publicationHandler.stats(), clusterStateStats); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 13033b670d44b..aca8653be4dd8 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -513,10 +513,10 @@ private static void ensureRemoteClusterStateNodesCompatibility(DiscoveryNode joi assert existingNodes.isEmpty() == false; Optional remotePublicationNode = existingNodes.stream() - .filter(DiscoveryNode::isRemoteStatePublicationEnabled) + .filter(DiscoveryNode::isRemoteStatePublicationConfigured) .findFirst(); - if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationEnabled()) { + if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationConfigured()) { ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES); } } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index caed2b6eceb49..6277b70f9a7c8 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -178,14 +178,6 @@ public PublishClusterStateStats stats() { ); } - public PersistedStateStats getFullDownloadStats() { - return remoteClusterStateService.getFullDownloadStats(); - } - - public PersistedStateStats getDiffDownloadStats() { - return remoteClusterStateService.getDiffDownloadStats(); - } - private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportRequest request) throws IOException { try (StreamInput in = CompressedStreamUtils.decompressBytes(request, namedWriteableRegistry)) { ClusterState incomingState; @@ -356,7 +348,7 @@ public PublicationContext newPublicationContext( ) { if (isRemotePublicationEnabled == true) { if (allNodesRemotePublicationEnabled.get() == false) { - if (validateRemotePublicationOnAllNodes(clusterChangedEvent.state().nodes()) == true) { + if (validateRemotePublicationConfiguredOnAllNodes(clusterChangedEvent.state().nodes()) == true) { allNodesRemotePublicationEnabled.set(true); } } @@ -374,11 +366,11 @@ public PublicationContext newPublicationContext( return publicationContext; } - private boolean validateRemotePublicationOnAllNodes(DiscoveryNodes discoveryNodes) { + private boolean validateRemotePublicationConfiguredOnAllNodes(DiscoveryNodes discoveryNodes) { assert ClusterMetadataManifest.getCodecForVersion(discoveryNodes.getMinNodeVersion()) >= ClusterMetadataManifest.CODEC_V0; for (DiscoveryNode node : discoveryNodes.getNodes().values()) { // if a node is non-remote then created local publication context - if (node.isRemoteStatePublicationEnabled() == false) { + if (node.isRemoteStatePublicationConfigured() == false) { return false; } } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index a6f0a457f7f9b..b06c2ef0bdbe4 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -515,10 +515,10 @@ public boolean isRemoteStoreNode() { } /** - * Returns whether remote cluster state publication is enabled on this node + * Returns whether settings required for remote cluster state publication is configured * @return true if the node contains remote cluster state node attribute and remote routing table node attribute */ - public boolean isRemoteStatePublicationEnabled() { + public boolean isRemoteStatePublicationConfigured() { return this.getAttributes() .keySet() .stream() diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index ce5e57b79dadb..1d7200792442f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -111,7 +111,7 @@ import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_METADATA_PREFIX; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteClusterStateConfigured; /** * A Service which provides APIs to upload and download cluster metadata from remote store. @@ -256,7 +256,7 @@ public RemoteClusterStateService( List indexMetadataUploadListeners, NamedWriteableRegistry namedWriteableRegistry ) { - assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled"; + assert isRemoteClusterStateConfigured(settings) : "Remote cluster state is not configured"; this.nodeId = nodeId; this.repositoriesService = repositoriesService; this.settings = settings; @@ -1061,7 +1061,7 @@ public void close() throws IOException { } public void start() { - assert isRemoteStoreClusterStateEnabled(settings) == true : "Remote cluster state is not enabled"; + assert isRemoteClusterStateConfigured(settings) == true : "Remote cluster state is not enabled"; final String remoteStoreRepo = settings.get( Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY ); diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java index 5878dff03acc2..2a76a5b966884 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java @@ -51,8 +51,8 @@ import static org.opensearch.index.remote.RemoteIndexPath.SEGMENT_PATH; import static org.opensearch.index.remote.RemoteIndexPath.TRANSLOG_PATH; import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStorePathStrategy; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteClusterStateConfigured; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** * Uploads the remote store path for all possible combinations of {@link org.opensearch.index.remote.RemoteStoreEnums.DataCategory} @@ -235,7 +235,7 @@ private Repository validateAndGetRepository(String repoSetting) { } public void start() { - assert isRemoteStoreClusterStateEnabled(settings) == true : "Remote cluster state is not enabled"; + assert isRemoteClusterStateConfigured(settings) == true : "Remote cluster state is not configured"; if (isRemoteDataAttributePresent == false) { // If remote store data attributes are not present than we skip this. return; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 56d04df5921ee..fe4bfa1e64ba1 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -311,9 +311,9 @@ import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteClusterStateConfigured; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreAttributePresent; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** * A node represent a node within a cluster ({@code cluster.name}). The {@link #client()} can be used @@ -792,7 +792,7 @@ protected Node( final RemoteClusterStateService remoteClusterStateService; final RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; final RemoteIndexPathUploader remoteIndexPathUploader; - if (isRemoteStoreClusterStateEnabled(settings)) { + if (isRemoteClusterStateConfigured(settings)) { remoteIndexPathUploader = new RemoteIndexPathUploader( threadPool, settings, diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 55971398634c5..d6a58f8e1d471 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -180,7 +180,7 @@ public static boolean isRemoteDataAttributePresent(Settings settings) { || settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false; } - public static boolean isRemoteClusterStateAttributePresent(Settings settings) { + public static boolean isRemoteClusterStateConfigured(Settings settings) { return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY) .isEmpty() == false; } @@ -194,8 +194,7 @@ public static String getRemoteStoreTranslogRepo(Settings settings) { } public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { - return RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) - && isRemoteClusterStateAttributePresent(settings); + return RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) && isRemoteClusterStateConfigured(settings); } private static boolean isRemoteRoutingTableAttributePresent(Settings settings) { diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 68a2b8086a92e..e27ff311c06f6 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2795,6 +2795,26 @@ public static Settings buildRemoteStoreNodeAttributes( ); } + public static Settings buildRemoteStateNodeAttributes(String stateRepoName, Path stateRepoPath, String stateRepoType) { + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + stateRepoName + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + stateRepoName + ); + String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey(); + Settings.Builder settings = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, stateRepoName) + .put(stateRepoTypeAttributeKey, stateRepoType) + .put(stateRepoSettingsAttributeKeyPrefix + "location", stateRepoPath) + .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable); + return settings.build(); + } + private static Settings buildRemoteStoreNodeAttributes( String segmentRepoName, Path segmentRepoPath, @@ -2850,16 +2870,6 @@ private static Settings buildRemoteStoreNodeAttributes( "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, translogRepoName ); - String stateRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - segmentRepoName - ); - String stateRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - segmentRepoName - ); String routingTableRepoAttributeKey = null, routingTableRepoSettingsAttributeKeyPrefix = null; if (routingTableRepoName != null) { routingTableRepoAttributeKey = String.format( @@ -2885,10 +2895,7 @@ private static Settings buildRemoteStoreNodeAttributes( .put(translogRepoTypeAttributeKey, translogRepoType) .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) .put(translogRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable) - .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) - .put(stateRepoTypeAttributeKey, segmentRepoType) - .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) - .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable); + .put(buildRemoteStateNodeAttributes(segmentRepoName, segmentRepoPath, segmentRepoType)); if (routingTableRepoName != null) { settings.put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName) .put(routingTableRepoAttributeKey, routingTableRepoType) From 6020c58659c1a7d8d9a65ae449b6c4b411a74d72 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Thu, 3 Oct 2024 17:17:16 +0530 Subject: [PATCH 098/108] [Snapshot V2] Remove orphan timestamps post create snapshot completion (#16148) Signed-off-by: Gaurav Bafna --- .../remotestore/RemoteRestoreSnapshotIT.java | 134 +++++++++++++++--- .../RemoteStorePinnedTimestampsIT.java | 14 +- .../RemoteStorePinnedTimestampService.java | 30 ++++ .../snapshots/SnapshotsService.java | 58 ++++++++ 4 files changed, 216 insertions(+), 20 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index a0183e89bfce2..927dbf9995778 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -9,6 +9,7 @@ package org.opensearch.remotestore; import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.LatchedActionListener; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; @@ -29,6 +30,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.index.Index; import org.opensearch.core.rest.RestStatus; @@ -41,6 +43,7 @@ import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryData; @@ -62,8 +65,10 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -73,6 +78,7 @@ import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING; +import static org.opensearch.snapshots.SnapshotsService.getPinningEntity; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -753,17 +759,15 @@ public void testInvalidRestoreRequestScenarios() throws Exception { assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore")); } - public void testCreateSnapshotV2() throws Exception { + public void testCreateSnapshotV2_Orphan_Timestamp_Cleanup() throws Exception { internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); internalCluster().startDataOnlyNode(pinnedTimestampSettings()); internalCluster().startDataOnlyNode(pinnedTimestampSettings()); String indexName1 = "testindex1"; String indexName2 = "testindex2"; - String indexName3 = "testindex3"; String snapshotRepoName = "test-create-snapshot-repo"; String snapshotName1 = "test-create-snapshot1"; Path absolutePath1 = randomRepoPath().toAbsolutePath(); - logger.info("Snapshot Path [{}]", absolutePath1); Settings.Builder settings = Settings.builder() .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) @@ -787,27 +791,37 @@ public void testCreateSnapshotV2() throws Exception { indexDocuments(client, indexName2, numDocsInIndex2); ensureGreen(indexName1, indexName2); + // create an orphan timestamp related to this repo + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + internalCluster().getClusterManagerName() + ); + forceSyncPinnedTimestamps(); + + long pinnedTimestamp = System.currentTimeMillis(); + final CountDownLatch latch = new CountDownLatch(1); + LatchedActionListener latchedActionListener = new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(Void unused) {} + + @Override + public void onFailure(Exception e) {} + }, latch); + + remoteStorePinnedTimestampService.pinTimestamp( + pinnedTimestamp, + getPinningEntity(snapshotRepoName, "some_uuid"), + latchedActionListener + ); + latch.await(); + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, Collections.emptyList()); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfo.successfulShards(), greaterThan(0)); assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L)); - indexDocuments(client, indexName1, 10); - indexDocuments(client, indexName2, 20); - - createIndex(indexName3, indexSettings); - indexDocuments(client, indexName3, 10); - - String snapshotName2 = "test-create-snapshot2"; - - // verify response status if waitForCompletion is not true - RestStatus createSnapshotResponseStatus = client().admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName2) - .get() - .status(); - assertEquals(RestStatus.ACCEPTED, createSnapshotResponseStatus); + waitUntil(() -> 1 == RemoteStorePinnedTimestampService.getPinnedEntities().size()); } public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception { @@ -879,7 +893,8 @@ public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2)); assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L)); - + forceSyncPinnedTimestamps(); + assertEquals(RemoteStorePinnedTimestampService.getPinnedEntities().size(), 1); } public void testConcurrentSnapshotV2CreateOperation() throws InterruptedException, ExecutionException { @@ -955,6 +970,8 @@ public void testConcurrentSnapshotV2CreateOperation() throws InterruptedExceptio RepositoryData repositoryData = repositoryDataPlainActionFuture.get(); assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1)); + forceSyncPinnedTimestamps(); + assertEquals(RemoteStorePinnedTimestampService.getPinnedEntities().size(), repositoryData.getSnapshotIds().size()); } public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Exception { @@ -1017,13 +1034,92 @@ public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Except logger.info("Exception while creating new-snapshot", e); } + AtomicLong totalSnaps = new AtomicLong(); + // Validate that snapshot is present in repository data assertBusy(() -> { GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName); GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet(); assertThat(response2.getSnapshots().size(), greaterThanOrEqualTo(1)); + totalSnaps.set(response2.getSnapshots().size()); + }, 30, TimeUnit.SECONDS); thread.join(); + forceSyncPinnedTimestamps(); + waitUntil(() -> { + this.forceSyncPinnedTimestamps(); + return RemoteStorePinnedTimestampService.getPinnedEntities().size() == totalSnaps.intValue(); + }); + } + + public void testCreateSnapshotV2() throws Exception { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String indexName3 = "testindex3"; + String snapshotRepoName = "test-create-snapshot-repo"; + String snapshotName1 = "test-create-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + + createRepository(snapshotRepoName, FsRepository.TYPE, settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, Collections.emptyList()); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L)); + + indexDocuments(client, indexName1, 10); + indexDocuments(client, indexName2, 20); + + createIndex(indexName3, indexSettings); + indexDocuments(client, indexName3, 10); + + String snapshotName2 = "test-create-snapshot2"; + + // verify response status if waitForCompletion is not true + RestStatus createSnapshotResponseStatus = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName2) + .get() + .status(); + assertEquals(RestStatus.ACCEPTED, createSnapshotResponseStatus); + forceSyncPinnedTimestamps(); + assertEquals(2, RemoteStorePinnedTimestampService.getPinnedEntities().size()); + } + + public void forceSyncPinnedTimestamps() { + // for all nodes , run forceSyncPinnedTimestamps() + for (String node : internalCluster().getNodeNames()) { + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + node + ); + remoteStorePinnedTimestampService.forceSyncPinnedTimestamps(); + } } public void testCreateSnapshotV2WithRedIndex() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java index 024e0e952eea5..3e1127e0ce240 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java @@ -19,6 +19,9 @@ import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.test.OpenSearchIntegTestCase; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -76,6 +79,11 @@ public void testTimestampPinUnpin() throws Exception { Tuple> pinnedTimestampWithFetchTimestamp_2 = RemoteStorePinnedTimestampService.getPinnedTimestamps(); long lastFetchTimestamp_2 = pinnedTimestampWithFetchTimestamp_2.v1(); assertTrue(lastFetchTimestamp_2 != -1); + Map> pinnedEntities = RemoteStorePinnedTimestampService.getPinnedEntities(); + assertEquals(3, pinnedEntities.size()); + assertEquals(Set.of("ss2", "ss3", "ss4"), pinnedEntities.keySet()); + assertEquals(pinnedEntities.get("ss2").size(), 1); + assertEquals(Optional.of(timestamp1).get(), pinnedEntities.get("ss2").get(0)); assertEquals(Set.of(timestamp1, timestamp2, timestamp3), pinnedTimestampWithFetchTimestamp_2.v2()); }); @@ -103,10 +111,14 @@ public void onFailure(Exception e) { // Adding different entity to already pinned timestamp remoteStorePinnedTimestampService.pinTimestamp(timestamp3, "ss5", noOpActionListener); - remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + remoteStorePinnedTimestampService.forceSyncPinnedTimestamps(); assertBusy(() -> { Tuple> pinnedTimestampWithFetchTimestamp_3 = RemoteStorePinnedTimestampService.getPinnedTimestamps(); + Map> pinnedEntities = RemoteStorePinnedTimestampService.getPinnedEntities(); + assertEquals(3, pinnedEntities.size()); + assertEquals(pinnedEntities.get("ss5").size(), 1); + assertEquals(Optional.of(timestamp3).get(), pinnedEntities.get("ss5").get(0)); long lastFetchTimestamp_3 = pinnedTimestampWithFetchTimestamp_3.v1(); assertTrue(lastFetchTimestamp_3 != -1); assertEquals(Set.of(timestamp1, timestamp3), pinnedTimestampWithFetchTimestamp_3.v2()); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index 1448c46583f6a..a3382d8568ec5 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -30,6 +30,9 @@ import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -49,6 +52,7 @@ public class RemoteStorePinnedTimestampService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteStorePinnedTimestampService.class); private static Tuple> pinnedTimestampsSet = new Tuple<>(-1L, Set.of()); + private static Map> pinnedEntityToTimestampsMap = new HashMap<>(); public static final String PINNED_TIMESTAMPS_PATH_TOKEN = "pinned_timestamps"; public static final String PINNED_TIMESTAMPS_FILENAME_SEPARATOR = "__"; @@ -216,6 +220,16 @@ private long getTimestampFromBlobName(String blobName) { return -1; } + private String getEntityFromBlobName(String blobName) { + String[] blobNameTokens = blobName.split(PINNED_TIMESTAMPS_FILENAME_SEPARATOR); + if (blobNameTokens.length < 2) { + String errorMessage = "Pinned timestamps blob name contains invalid format: " + blobName; + logger.error(errorMessage); + throw new IllegalArgumentException(errorMessage); + } + return String.join(PINNED_TIMESTAMPS_FILENAME_SEPARATOR, Arrays.copyOfRange(blobNameTokens, 0, blobNameTokens.length - 1)); + } + /** * Unpins a timestamp from the remote store. * @@ -262,6 +276,10 @@ public static Tuple> getPinnedTimestamps() { return pinnedTimestampsSet; } + public static Map> getPinnedEntities() { + return pinnedEntityToTimestampsMap; + } + /** * Inner class for asynchronously updating the pinned timestamp set. */ @@ -283,6 +301,7 @@ protected void runInternal() { Map pinnedTimestampList = blobContainer.listBlobs(); if (pinnedTimestampList.isEmpty()) { pinnedTimestampsSet = new Tuple<>(triggerTimestamp, Set.of()); + pinnedEntityToTimestampsMap = new HashMap<>(); return; } Set pinnedTimestamps = pinnedTimestampList.keySet() @@ -290,8 +309,19 @@ protected void runInternal() { .map(RemoteStorePinnedTimestampService.this::getTimestampFromBlobName) .filter(timestamp -> timestamp != -1) .collect(Collectors.toSet()); + logger.debug("Fetched pinned timestamps from remote store: {} - {}", triggerTimestamp, pinnedTimestamps); pinnedTimestampsSet = new Tuple<>(triggerTimestamp, pinnedTimestamps); + pinnedEntityToTimestampsMap = pinnedTimestampList.keySet() + .stream() + .collect(Collectors.toMap(RemoteStorePinnedTimestampService.this::getEntityFromBlobName, blobName -> { + long timestamp = RemoteStorePinnedTimestampService.this.getTimestampFromBlobName(blobName); + return Collections.singletonList(timestamp); + }, (existingList, newList) -> { + List mergedList = new ArrayList<>(existingList); + mergedList.addAll(newList); + return mergedList; + })); } catch (Throwable t) { logger.error("Exception while fetching pinned timestamp details", t); } diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 22b2a72b36026..c80f18cdd82f7 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -38,6 +38,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.Version; import org.opensearch.action.ActionRunnable; +import org.opensearch.action.LatchedActionListener; import org.opensearch.action.StepListener; import org.opensearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; @@ -621,6 +622,7 @@ public void onResponse(RepositoryData repositoryData) { return; } listener.onResponse(snapshotInfo); + cleanOrphanTimestamp(repositoryName, repositoryData); } @Override @@ -651,6 +653,57 @@ public TimeValue timeout() { }, "create_snapshot [" + snapshotName + ']', listener::onFailure); } + private void cleanOrphanTimestamp(String repoName, RepositoryData repositoryData) { + Collection snapshotUUIDs = repositoryData.getSnapshotIds().stream().map(SnapshotId::getUUID).collect(Collectors.toSet()); + Map> pinnedEntities = RemoteStorePinnedTimestampService.getPinnedEntities(); + + List orphanPinnedEntities = pinnedEntities.keySet() + .stream() + .filter(pinnedEntity -> isOrphanPinnedEntity(repoName, snapshotUUIDs, pinnedEntity)) + .collect(Collectors.toList()); + + if (orphanPinnedEntities.isEmpty()) { + return; + } + + logger.info("Found {} orphan timestamps. Cleaning it up now", orphanPinnedEntities.size()); + if (tryEnterRepoLoop(repoName)) { + deleteOrphanTimestamps(pinnedEntities, orphanPinnedEntities); + leaveRepoLoop(repoName); + } else { + logger.info("Concurrent snapshot create/delete is happening. Skipping clean up of orphan timestamps"); + } + } + + private boolean isOrphanPinnedEntity(String repoName, Collection snapshotUUIDs, String pinnedEntity) { + Tuple tokens = getRepoSnapshotUUIDTuple(pinnedEntity); + return Objects.equals(tokens.v1(), repoName) && snapshotUUIDs.contains(tokens.v2()) == false; + } + + private void deleteOrphanTimestamps(Map> pinnedEntities, List orphanPinnedEntities) { + final CountDownLatch latch = new CountDownLatch(orphanPinnedEntities.size()); + for (String pinnedEntity : orphanPinnedEntities) { + assert pinnedEntities.get(pinnedEntity).size() == 1 : "Multiple timestamps for same repo-snapshot uuid found"; + long orphanTimestamp = pinnedEntities.get(pinnedEntity).get(0); + remoteStorePinnedTimestampService.unpinTimestamp( + orphanTimestamp, + pinnedEntity, + new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(Void unused) {} + + @Override + public void onFailure(Exception e) {} + }, latch) + ); + } + try { + latch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + private void createSnapshotPreValidations( ClusterState currentState, RepositoryData repositoryData, @@ -707,6 +760,11 @@ public static String getPinningEntity(String repositoryName, String snapshotUUID return repositoryName + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshotUUID; } + public static Tuple getRepoSnapshotUUIDTuple(String pinningEntity) { + String[] tokens = pinningEntity.split(SNAPSHOT_PINNED_TIMESTAMP_DELIMITER); + return new Tuple<>(tokens[0], tokens[1]); + } + private void cloneSnapshotPinnedTimestamp( RepositoryData repositoryData, SnapshotId sourceSnapshot, From 3d7184b03326d101be737b9a40183a55fb6287c1 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 3 Oct 2024 14:29:44 -0400 Subject: [PATCH 099/108] Bump Netty to 4.1.114.Final (#16182) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- .../licenses/netty-buffer-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-common-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-common-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.112.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-socks-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-socks-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.112.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 + .../repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 | 1 - .../repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 + .../netty-transport-classes-epoll-4.1.112.Final.jar.sha1 | 1 - .../netty-transport-classes-epoll-4.1.114.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.112.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-buffer-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-common-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-common-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.112.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 + 90 files changed, 46 insertions(+), 45 deletions(-) delete mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index b69c33df3b6a5..59bcb448bbef2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) +- Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 63817289e80c0..8cede6b717883 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -29,7 +29,7 @@ hdrhistogram = 2.2.2 # when updating the JNA version, also update the version in buildSrc/build.gradle jna = 5.13.0 -netty = 4.1.112.Final +netty = 4.1.114.Final joda = 2.12.7 # project reactor diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5c26883046fed..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bdc12df04bb6858890b8aa108060b5b365a26102 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..8cb83fc367d78 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1fd224fdd0b44..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..1be26fee34d46 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 deleted file mode 100644 index 22d35128c3ad5..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81af1040bfa977f98dd0e1bd9639513ea862ca04 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e683773245716 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 deleted file mode 100644 index d4767d06b22bf..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fa28b510f0f16f4d5d7188b86bef59e048f62f9 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..2c18924e33c62 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 47af3100f0f2d..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2798069092a981a832b7510d0462ee9efb7a80e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..74ce939dc6190 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8b30272861770..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d5e2d5bcc6baeeb8c13a230980c6132a778e036 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..c431976b6fbd2 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1a094fa19a623..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58a631d9d44c4ed7cc0dcc9cffa6641da9374d72 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e8080a5b2acb1 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5fbfde0836e0c..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77cd136dd3843f5e7cbcf68c824975d745c49ddb \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..fb2d518789a18 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8dad0e3104dc8..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b50ff619cdcdc48e748cba3405c9988529f28f60 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..a80b9e51be74b --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 deleted file mode 100644 index a42a41b6387c8..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -06724b184ee870ecc4d8fc36931beeb3c387b0ee \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..04a48547adb05 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +35798729ba06670fb4fcd02db98d9577e363992d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 deleted file mode 100644 index d4767d06b22bf..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fa28b510f0f16f4d5d7188b86bef59e048f62f9 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..2c18924e33c62 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5291a16c10448..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9aed7e78c467d06a47a45b5b27466380a6427e2f \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..acbab117e4c15 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +42b1159cac3d196f6bdbd528e29f0fab9dbaae06 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 deleted file mode 100644 index cf50574b87da0..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b23c87a85451b3b0e7c3e8e89698cea6831a8418 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..891ed0a444dc2 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +a01071edffb4812009312b461ce5f160cdec9b75 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 deleted file mode 100644 index 24e8177190e04..0000000000000 --- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -375872f1c16bb51aac016ff6ee4f5d28b1288d4d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..7df7a05cd7345 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +2fed36ff50059ded641fa5064963c4b4313512f3 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8dad0e3104dc8..0000000000000 --- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b50ff619cdcdc48e748cba3405c9988529f28f60 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..a80b9e51be74b --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 deleted file mode 100644 index 7c36b789e839c..0000000000000 --- a/plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d6b2e543749a86957777a46cf68aaa337cc558cb \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..32022e1b2eaa4 --- /dev/null +++ b/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +08134b298d48eec5ddf2e3674a978b52e4667304 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5c26883046fed..0000000000000 --- a/plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bdc12df04bb6858890b8aa108060b5b365a26102 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..8cb83fc367d78 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1fd224fdd0b44..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..1be26fee34d46 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1 deleted file mode 100644 index 22d35128c3ad5..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81af1040bfa977f98dd0e1bd9639513ea862ca04 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e683773245716 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 deleted file mode 100644 index d4767d06b22bf..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fa28b510f0f16f4d5d7188b86bef59e048f62f9 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..2c18924e33c62 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 47af3100f0f2d..0000000000000 --- a/plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2798069092a981a832b7510d0462ee9efb7a80e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..74ce939dc6190 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8b30272861770..0000000000000 --- a/plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d5e2d5bcc6baeeb8c13a230980c6132a778e036 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..c431976b6fbd2 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1a094fa19a623..0000000000000 --- a/plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58a631d9d44c4ed7cc0dcc9cffa6641da9374d72 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e8080a5b2acb1 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5fbfde0836e0c..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77cd136dd3843f5e7cbcf68c824975d745c49ddb \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..fb2d518789a18 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 deleted file mode 100644 index 0196dacfe92ba..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -67e590356eb53c20aaabd67f61ae66f628e62e3d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..7120cd93e5c0d --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f442c794e6fe89e6974f058bf393353e01fb927d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8dad0e3104dc8..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b50ff619cdcdc48e748cba3405c9988529f28f60 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..a80b9e51be74b --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5c26883046fed..0000000000000 --- a/plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bdc12df04bb6858890b8aa108060b5b365a26102 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..8cb83fc367d78 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1fd224fdd0b44..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..1be26fee34d46 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1 deleted file mode 100644 index 22d35128c3ad5..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81af1040bfa977f98dd0e1bd9639513ea862ca04 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e683773245716 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 47af3100f0f2d..0000000000000 --- a/plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2798069092a981a832b7510d0462ee9efb7a80e \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..74ce939dc6190 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8b30272861770..0000000000000 --- a/plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d5e2d5bcc6baeeb8c13a230980c6132a778e036 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..c431976b6fbd2 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1a094fa19a623..0000000000000 --- a/plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58a631d9d44c4ed7cc0dcc9cffa6641da9374d72 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e8080a5b2acb1 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5fbfde0836e0c..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77cd136dd3843f5e7cbcf68c824975d745c49ddb \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..fb2d518789a18 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5c26883046fed..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bdc12df04bb6858890b8aa108060b5b365a26102 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..8cb83fc367d78 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1fd224fdd0b44..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..1be26fee34d46 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 deleted file mode 100644 index a42a41b6387c8..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -06724b184ee870ecc4d8fc36931beeb3c387b0ee \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..04a48547adb05 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +35798729ba06670fb4fcd02db98d9577e363992d \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 deleted file mode 100644 index 22d35128c3ad5..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81af1040bfa977f98dd0e1bd9639513ea862ca04 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e683773245716 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 deleted file mode 100644 index d4767d06b22bf..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fa28b510f0f16f4d5d7188b86bef59e048f62f9 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..2c18924e33c62 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 47af3100f0f2d..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2798069092a981a832b7510d0462ee9efb7a80e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..74ce939dc6190 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8b30272861770..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d5e2d5bcc6baeeb8c13a230980c6132a778e036 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..c431976b6fbd2 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1a094fa19a623..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58a631d9d44c4ed7cc0dcc9cffa6641da9374d72 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e8080a5b2acb1 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 deleted file mode 100644 index 24e8177190e04..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -375872f1c16bb51aac016ff6ee4f5d28b1288d4d \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..7df7a05cd7345 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +2fed36ff50059ded641fa5064963c4b4313512f3 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5fbfde0836e0c..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77cd136dd3843f5e7cbcf68c824975d745c49ddb \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..fb2d518789a18 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8dad0e3104dc8..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b50ff619cdcdc48e748cba3405c9988529f28f60 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..a80b9e51be74b --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file From d1fd47c652b4c6a2c0ec5d0ee574a0ff0d263177 Mon Sep 17 00:00:00 2001 From: Ralph Ursprung <39383228+rursprung@users.noreply.github.com> Date: Fri, 4 Oct 2024 00:39:19 +0200 Subject: [PATCH 100/108] Implement phone number analyzer (#15915) * add `Strings#isDigits` API inspiration taken from [this SO answer][SO]. note that the stream is not parallelised to avoid the overhead of this as the method is intended to be called primarily with shorter strings where the time to set up would take longer than the actual check. [SO]: https://stackoverflow.com/a/35150400 Signed-off-by: Ralph Ursprung * add `phone` & `phone-search` analyzer + tokenizer this is largely based on [elasticsearch-phone] and internally uses [libphonenumber]. this intentionally only ports a subset of the features: only `phone` and `phone-search` are supported right now, `phone-email` can be added if/when there's a clear need for it. using `libphonenumber` is required since parsing phone numbers is a non-trivial task (even though it might seem trivial at first glance!), as can be seen in the list [falsehoods programmers believe about phone numbers][falsehoods]. this allows defining the region to be used when analysing a phone number. so far only the generic "unkown" region (`ZZ`) had been used which worked as long as international numbers were prefixed with `+` but did not work when using local numbers (e.g. a number stored as `+4158...` was not matched against a number entered as `004158...` or `058...`). example configuration for an index: ```json { "index": { "analysis": { "analyzer": { "phone": { "type": "phone" }, "phone-search": { "type": "phone-search" }, "phone-ch": { "type": "phone", "phone-region": "CH" }, "phone-search-ch": { "type": "phone-search", "phone-region": "CH" } } } } } ``` this creates four analyzers: `phone` and `phone-search` which do not explicitly specify a region and thus fall back to `ZZ` (unknown region, regional version of international dialing prefix (e.g. `00` instead of `+` in most of europe) will not be recognised) and `phone-ch` and `phone-search-ch` which will try to parse the phone number as a swiss phone number (thus e.g. `00` as a prefix is recognised as the international dialing prefix). note that the analyzer is (currently) not meant to find phone numbers in large text documents - instead it should be used on fields which contain just the phone number (though extra text will be ignored) and it collects the whole content of the field into a `String` in memory, making it unsuitable for large field values. this has been implemented in a new plugin which is however part of the central opensearch repository as it was deemed too big an overhead to have it in a separate repository but not important enough to bundle it directly in `analysis-common` (see the discussion on the issue and the PR for further details). note that the new plugin has been added to the exclude list of the javadoc check as this check is overzealous and also complains in many cases where it shouldn't (e.g. on overridden methods - which it should theoretically not do - or constructors which don't even exist). the check first needs to be improved before this exclusion could be removed. closes opensearch-project/OpenSearch#11326 [elasticsearch-phone]: https://github.com/purecloudlabs/elasticsearch-phone [libphonenumber]: https://github.com/google/libphonenumber [falsehoods]: https://github.com/google/libphonenumber/blob/master/FALSEHOODS.md Signed-off-by: Ralph Ursprung --------- Signed-off-by: Ralph Ursprung --- CHANGELOG.md | 1 + gradle/missing-javadoc.gradle | 1 + .../org/opensearch/core/common/Strings.java | 13 + .../opensearch/core/common/StringsTests.java | 11 + plugins/analysis-phonenumber/build.gradle | 21 ++ .../licenses/libphonenumber-8.13.45.jar.sha1 | 1 + .../licenses/libphonenumber-LICENSE.txt | 176 ++++++++++++ .../licenses/libphonenumber-NOTICE.txt | 0 .../phone/PhoneNumberAnalysisPlugin.java | 60 +++++ .../analysis/phone/PhoneNumberAnalyzer.java | 51 ++++ .../phone/PhoneNumberAnalyzerProvider.java | 42 +++ .../phone/PhoneNumberTermTokenizer.java | 157 +++++++++++ .../PhoneNumberTermTokenizerFactory.java | 44 +++ .../phone/PhoneNumberAnalyzerTests.java | 253 ++++++++++++++++++ .../analysis/phone/phone_analysis.json | 22 ++ ...neNumberAnalysisClientYamlTestSuiteIT.java | 49 ++++ .../test/analysis-phone/10_basic.yml | 8 + .../test/analysis-phone/20_search.yml | 56 ++++ 18 files changed, 966 insertions(+) create mode 100644 plugins/analysis-phonenumber/build.gradle create mode 100644 plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1 create mode 100644 plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt create mode 100644 plugins/analysis-phonenumber/licenses/libphonenumber-NOTICE.txt create mode 100644 plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java create mode 100644 plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java create mode 100644 plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java create mode 100644 plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java create mode 100644 plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java create mode 100644 plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java create mode 100644 plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json create mode 100644 plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java create mode 100644 plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml create mode 100644 plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 59bcb448bbef2..ca23cd59e06d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) - [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) - Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986)) +- New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/gradle/missing-javadoc.gradle b/gradle/missing-javadoc.gradle index e9a6d798b8323..26898673bf608 100644 --- a/gradle/missing-javadoc.gradle +++ b/gradle/missing-javadoc.gradle @@ -127,6 +127,7 @@ configure([ project(":plugins:analysis-icu"), project(":plugins:analysis-kuromoji"), project(":plugins:analysis-nori"), + project(":plugins:analysis-phonenumber"), project(":plugins:analysis-phonetic"), project(":plugins:analysis-smartcn"), project(":plugins:analysis-stempel"), diff --git a/libs/core/src/main/java/org/opensearch/core/common/Strings.java b/libs/core/src/main/java/org/opensearch/core/common/Strings.java index 8fdec670bd9f2..e8379e11ea26a 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/Strings.java +++ b/libs/core/src/main/java/org/opensearch/core/common/Strings.java @@ -815,4 +815,17 @@ public static String toLowercaseAscii(String in) { } return out.toString(); } + + /** + * Check whether every single character in the string is a digit. + * + *

An empty string returns {@code false}.

+ * + * @param s the string, must not be null. + * @return {@code true} if the string only contains digits, {@code false} otherwise. + */ + public static boolean isDigits(final String s) { + return !s.isEmpty() && s.chars().allMatch(Character::isDigit); + } + } diff --git a/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java b/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java index b79bb6fc89f9e..be7af18b106a8 100644 --- a/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java @@ -114,4 +114,15 @@ public void testToStringToXContentWithOrWithoutParams() { containsString("\"color_from_param\":\"blue\"") ); } + + public void testIsDigits() { + assertTrue(Strings.isDigits("1")); + assertTrue(Strings.isDigits("123")); + assertFalse(Strings.isDigits("")); + assertFalse(Strings.isDigits("abc")); + assertFalse(Strings.isDigits("123a")); + assertFalse(Strings.isDigits("0x123")); + assertFalse(Strings.isDigits("123.4")); + assertFalse(Strings.isDigits("123f")); + } } diff --git a/plugins/analysis-phonenumber/build.gradle b/plugins/analysis-phonenumber/build.gradle new file mode 100644 index 0000000000000..c9913b36f8508 --- /dev/null +++ b/plugins/analysis-phonenumber/build.gradle @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +apply plugin: 'opensearch.yaml-rest-test' + +opensearchplugin { + description 'Adds an analyzer for phone numbers to OpenSearch.' + classname 'org.opensearch.analysis.phone.PhoneNumberAnalysisPlugin' +} + +dependencies { + implementation group: 'com.googlecode.libphonenumber', name: 'libphonenumber', version: '8.13.45' +} diff --git a/plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1 b/plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1 new file mode 100644 index 0000000000000..00d393482ee49 --- /dev/null +++ b/plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1 @@ -0,0 +1 @@ +bfac00f71616796abc7d8b135dda12558a0ccee2 \ No newline at end of file diff --git a/plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt b/plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt new file mode 100644 index 0000000000000..d9a10c0d8e868 --- /dev/null +++ b/plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/plugins/analysis-phonenumber/licenses/libphonenumber-NOTICE.txt b/plugins/analysis-phonenumber/licenses/libphonenumber-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java new file mode 100644 index 0000000000000..eb12b43f70154 --- /dev/null +++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import org.apache.lucene.analysis.Analyzer; +import org.opensearch.index.analysis.AnalyzerProvider; +import org.opensearch.index.analysis.TokenizerFactory; +import org.opensearch.indices.analysis.AnalysisModule; +import org.opensearch.plugins.AnalysisPlugin; +import org.opensearch.plugins.Plugin; + +import java.util.Map; +import java.util.TreeMap; + +/** + * This plugin provides an analyzer and tokenizer for fields which contain phone numbers, supporting a variety of formats + * (with/without international calling code, different country formats, etc.). + */ +public class PhoneNumberAnalysisPlugin extends Plugin implements AnalysisPlugin { + + @Override + public Map>> getAnalyzers() { + Map>> analyzers = new TreeMap<>(); + analyzers.put( + "phone", + (indexSettings, environment, name, settings) -> new PhoneNumberAnalyzerProvider(indexSettings, "phone", settings, true) + ); + analyzers.put( + "phone-search", + (indexSettings, environment, name, settings) -> new PhoneNumberAnalyzerProvider(indexSettings, "phone-search", settings, false) + ); + return analyzers; + } + + @Override + public Map> getTokenizers() { + Map> tokenizers = new TreeMap<>(); + tokenizers.put( + "phone", + (indexSettings, environment, name, settings) -> new PhoneNumberTermTokenizerFactory(indexSettings, "phone", settings, true) + ); + tokenizers.put( + "phone-search", + (indexSettings, environment, name, settings) -> new PhoneNumberTermTokenizerFactory( + indexSettings, + "phone-search", + settings, + false + ) + ); + return tokenizers; + } + +} diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java new file mode 100644 index 0000000000000..cd945e186b2ba --- /dev/null +++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import org.apache.lucene.analysis.Analyzer; +import org.opensearch.common.settings.Settings; + +/** + * Analyzer for phone numbers, using {@link PhoneNumberTermTokenizer}. + * + *

+ * You can use the {@code phone} and {@code phone-search} analyzers on your fields to index phone numbers. + * Use {@code phone} (which creates ngrams) for the {@code analyzer} and {@code phone-search} (which doesn't create ngrams) + * for the {@code search_analyzer}. + *

+ * + *

+ * You optionally can specify a region with the {@code phone-region} setting for the phone number which will ensure that + * phone numbers without the international dialling prefix (using {@code +}) are also tokenized correctly. + *

+ * + *

+ * Note that the tokens will not refer to a specific position in the stream as the tokenizer is expected to be used on strings + * containing phone numbers and not arbitrary text with interspersed phone numbers. + *

+ */ +public class PhoneNumberAnalyzer extends Analyzer { + private final boolean addNgrams; + private final Settings settings; + + /** + * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search. + * @param settings the settings for the analyzer. + */ + public PhoneNumberAnalyzer(final Settings settings, final boolean addNgrams) { + this.addNgrams = addNgrams; + this.settings = settings; + } + + @Override + protected TokenStreamComponents createComponents(String fieldName) { + final var tokenizer = new PhoneNumberTermTokenizer(this.settings, this.addNgrams); + return new Analyzer.TokenStreamComponents(tokenizer, tokenizer); + } +} diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java new file mode 100644 index 0000000000000..272a019ba0f9c --- /dev/null +++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.analysis.AbstractIndexAnalyzerProvider; + +/** + * Provider for {@link PhoneNumberAnalyzer}. + */ +public class PhoneNumberAnalyzerProvider extends AbstractIndexAnalyzerProvider { + + private final PhoneNumberAnalyzer analyzer; + + /** + * @param indexSettings the settings of the index. + * @param name the analyzer name. + * @param settings the settings for the analyzer. + * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search. + */ + public PhoneNumberAnalyzerProvider( + final IndexSettings indexSettings, + final String name, + final Settings settings, + final boolean addNgrams + ) { + super(indexSettings, name, settings); + this.analyzer = new PhoneNumberAnalyzer(settings, addNgrams); + } + + @Override + public PhoneNumberAnalyzer get() { + return this.analyzer; + } +} diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java new file mode 100644 index 0000000000000..6b95594204eb4 --- /dev/null +++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import com.google.i18n.phonenumbers.NumberParseException; +import com.google.i18n.phonenumbers.PhoneNumberUtil; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.opensearch.common.io.Streams; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Optional; +import java.util.Set; + +/** + * This tokenizes a phone number into its individual parts, using {@link PhoneNumberUtil}. + * + *

+ * You can use the {@code phone} and {@code phone-search} analyzers on your fields to index phone numbers. + * Use {@code phone} (which creates ngrams) for the {@code analyzer} and {@code phone-search} (which doesn't create ngrams) + * for the {@code search_analyzer}. + *

+ * + *

+ * You optionally can specify a region with the {@code phone-region} setting for the phone number which will ensure that + * phone numbers without the international dialling prefix (using {@code +}) are also tokenized correctly. + *

+ * + *

+ * Note that the tokens will not refer to a specific position in the stream as the tokenizer is expected to be used on strings + * containing phone numbers and not arbitrary text with interspersed phone numbers. + *

+ */ +public final class PhoneNumberTermTokenizer extends Tokenizer { + private final boolean addNgrams; + private final Settings settings; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private Iterator tokenIterator; + + /** + * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search. + * @param settings the settings for the analyzer. + */ + public PhoneNumberTermTokenizer(final Settings settings, final boolean addNgrams) { + super(); + this.addNgrams = addNgrams; + this.settings = settings; + } + + /** {@inheritDoc} */ + @Override + public void reset() throws IOException { + super.reset(); + tokenIterator = null; + } + + /** {@inheritDoc} */ + @Override + public boolean incrementToken() throws IOException { + clearAttributes(); + if (tokenIterator == null) { + tokenIterator = getTokens().iterator(); + } + if (tokenIterator.hasNext()) { + termAtt.append(tokenIterator.next()); + return true; + } + return false; + } + + /** + * Search for a phone number in the input and tokenize it. + * + *

+ * The tokens include the full phone number with and without country prefix (if it could be identified) and - if + * enabled by {@link #addNgrams} - an ngram of the phone number. + *

+ * + * @return all tokens (unique, unordered). + * @throws IOException in case the input cannot be read. + */ + private Set getTokens() throws IOException { + final var tokens = new HashSet(); + + var input = Streams.copyToString(this.input); + + tokens.add(input); + + // Rip off the "tel:" or "sip:" prefix + if (input.indexOf("tel:") == 0 || input.indexOf("sip:") == 0) { + tokens.add(input.substring(0, 4)); + input = input.substring(4); + } + + final var startIndex = input.startsWith("+") ? 1 : 0; + // Add the complete input but skip a leading + + tokens.add(input.substring(startIndex)); + + // Drop anything after @. Most likely there's nothing of interest + final var posAt = input.indexOf('@'); + if (posAt != -1) { + input = input.substring(0, posAt); + + // Add a token for the raw unmanipulated address. Note this could be a username (sip) instead of telephone + // number so take it as is + tokens.add(input.substring(startIndex)); + } + + // Let google's libphone try to parse it + final var phoneUtil = PhoneNumberUtil.getInstance(); + Optional countryCode = Optional.empty(); + try { + // ZZ is the generic "I don't know the country code" region. Google's libphone library will try to infer it. + final var region = this.settings.get("phone-region", "ZZ"); + final var numberProto = phoneUtil.parse(input, region); + if (numberProto != null) { + // Libphone likes it! + countryCode = Optional.of(String.valueOf(numberProto.getCountryCode())); + input = String.valueOf(numberProto.getNationalNumber()); + + // Add Country code, extension, and the number as tokens + tokens.add(countryCode.get()); + tokens.add(countryCode.get() + input); + if (!Strings.isEmpty(numberProto.getExtension())) { + tokens.add(numberProto.getExtension()); + } + + tokens.add(input); + } + } catch (final NumberParseException | StringIndexOutOfBoundsException e) { + // Libphone didn't like it, no biggie. We'll just ngram the number as it is. + } + + // ngram the phone number, e.g. 19198243333 produces 9, 91, 919, etc + if (this.addNgrams && Strings.isDigits(input)) { + for (int count = 1; count <= input.length(); ++count) { + final var token = input.substring(0, count); + tokens.add(token); + // If there was a country code, add more ngrams such that 19198243333 produces 19, 191, 1919, etc + countryCode.ifPresent(s -> tokens.add(s + token)); + } + } + + return tokens; + } + +} diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java new file mode 100644 index 0000000000000..fde44e15c9667 --- /dev/null +++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import org.apache.lucene.analysis.Tokenizer; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.analysis.AbstractTokenizerFactory; + +/** + * Factory for {@link PhoneNumberTermTokenizer}. + */ +public class PhoneNumberTermTokenizerFactory extends AbstractTokenizerFactory { + private final Settings settings; + private final boolean addNgrams; + + /** + * @param indexSettings the settings of the index. + * @param name the tokenizer name. + * @param settings the settings for the analyzer. + * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search. + */ + public PhoneNumberTermTokenizerFactory( + final IndexSettings indexSettings, + final String name, + final Settings settings, + final boolean addNgrams + ) { + super(indexSettings, settings, name); + this.settings = settings; + this.addNgrams = addNgrams; + } + + @Override + public Tokenizer create() { + return new PhoneNumberTermTokenizer(this.settings, this.addNgrams); + } +} diff --git a/plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java b/plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java new file mode 100644 index 0000000000000..332f6d21f47d6 --- /dev/null +++ b/plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java @@ -0,0 +1,253 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.opensearch.index.analysis.AnalysisTestsHelper; +import org.opensearch.test.OpenSearchTokenStreamTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.hasItemInArray; + +public class PhoneNumberAnalyzerTests extends OpenSearchTokenStreamTestCase { + private static final String RESOURCE = "/org/opensearch/analysis/phone/phone_analysis.json"; + + private static Analyzer phoneAnalyzer; + private static Analyzer phoneSearchAnalyzer; + private static Analyzer phoneCHAnalyzer; + private static Analyzer phoneSearchCHAnalyzer; + + @BeforeClass + public static void beforeClass() throws IOException { + final var analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath( + createTempDir(), + RESOURCE, + new PhoneNumberAnalysisPlugin() + ); + phoneAnalyzer = analysis.indexAnalyzers.get("phone"); + assertNotNull(phoneAnalyzer); + phoneSearchAnalyzer = analysis.indexAnalyzers.get("phone-search"); + assertNotNull(phoneSearchAnalyzer); + phoneCHAnalyzer = analysis.indexAnalyzers.get("phone-ch"); + assertNotNull(phoneCHAnalyzer); + phoneSearchCHAnalyzer = analysis.indexAnalyzers.get("phone-search-ch"); + assertNotNull(phoneSearchCHAnalyzer); + } + + /** + * Test for all tokens which are emitted by the "phone" analyzer. + */ + public void testEuropeDetailled() throws IOException { + assertTokensAreInAnyOrder( + phoneAnalyzer, + "tel:+441344840400", + Arrays.asList( + "tel:+441344840400", + "tel:", + "441344840400", + "44", + "1344840400", + "1", + "441", + "13", + "4413", + "134", + "44134", + "1344", + "441344", + "13448", + "4413448", + "134484", + "44134484", + "1344840", + "441344840", + "13448404", + "4413448404", + "134484040", + "44134484040" + ) + ); + } + + /** + * Test for all tokens which are emitted by the "phone" analyzer. + */ + public void testEuropeDetailledSearch() throws IOException { + assertTokensAreInAnyOrder( + phoneSearchAnalyzer, + "tel:+441344840400", + Arrays.asList("tel:+441344840400", "tel:", "441344840400", "44", "1344840400") + ); + } + + public void testEurope() throws IOException { + assertTokensInclude("tel:+441344840400", Arrays.asList("44", "1344", "1344840400", "441344840400")); + } + + public void testGermanCastle() throws IOException { + assertTokensInclude("tel:+498362930830", Arrays.asList("49", "498362930830", "8362930830")); + } + + public void testBMWofSydney() throws IOException { + assertTokensInclude("tel:+61293344555", Arrays.asList("61", "293344555", "61293344555")); + } + + public void testCoffeeShopInIreland() throws IOException { + assertTokensInclude("tel:+442890319416", Arrays.asList("44", "289", "2890319416", "442890319416")); + } + + public void testTelWithCountryCode() throws IOException { + assertTokensInclude("tel:+17177158163", Arrays.asList("1", "717", "7177", "17177158163")); + } + + public void testTelWithCountryCode2() throws IOException { + assertTokensInclude("tel:+12177148350", Arrays.asList("1", "217", "2177", "2177148350", "12177148350")); + } + + public void testNewTollFreeNumber() throws IOException { + assertTokensInclude("tel:+18337148350", Arrays.asList("1", "833", "8337", "8337148350", "18337148350")); + } + + public void testMissingCountryCode() throws IOException { + assertTokensInclude("tel:8177148350", Arrays.asList("817", "8177", "81771", "817714", "8177148350")); + } + + public void testSipWithNumericUsername() throws IOException { + assertTokensInclude("sip:222@autosbcpc", Arrays.asList("222")); + } + + public void testTruncatedNumber() throws IOException { + assertTokensInclude("tel:5551234", Arrays.asList("5551234")); + } + + public void testSipWithAlphabeticUsername() throws IOException { + assertTokensInclude("sip:abc@autosbcpc", Arrays.asList("abc")); + } + + public void testGarbageInGarbageOut() throws IOException { + assertTokensInclude("test", Arrays.asList("test")); + } + + public void testSipWithCountryCode() throws IOException { + assertTokensInclude("sip:+14177141363@178.97.105.13;isup-oli=0;pstn-params=808481808882", Arrays.asList("417", "4177", "14177")); + } + + public void testSipWithTelephoneExtension() throws IOException { + assertTokensInclude("sip:+13169410766;ext=2233@178.17.10.117:8060", Arrays.asList("316", "2233", "1316")); + } + + public void testSipWithUsername() throws IOException { + assertTokensInclude("sip:JeffSIP@178.12.220.18", Arrays.asList("JeffSIP")); + } + + public void testPhoneNumberWithoutPrefix() throws IOException { + assertTokensInclude("+14177141363", Arrays.asList("14177141363", "417", "4177", "14177")); + } + + public void testSipWithoutDomainPart() throws IOException { + assertTokensInclude("sip:+122882", Arrays.asList("122882", "122", "228", "1228", "2288", "12288")); + } + + public void testTelPrefix() throws IOException { + assertTokensInclude("tel:+1228", Arrays.asList("1228", "122", "228")); + } + + public void testNumberPrefix() throws IOException { + assertTokensInclude("+1228", Arrays.asList("1228", "122", "228")); + } + + public void testInternationalPrefixWithZZ() throws IOException { + assertTokensInclude(phoneAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testInternationalPrefixWithCH() throws IOException { + assertTokensInclude(phoneCHAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testNationalPrefixWithCH() throws IOException { + // + is equivalent to 00 in Switzerland + assertTokensInclude(phoneCHAnalyzer, "0041583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testLocalNumberWithCH() throws IOException { + // when omitting the international prefix swiss numbers must start with '0' + assertTokensInclude(phoneCHAnalyzer, "0583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testSearchInternationalPrefixWithZZ() throws IOException { + assertTokensInclude(phoneSearchAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testSearchInternationalPrefixWithCH() throws IOException { + assertTokensInclude(phoneSearchCHAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testSearchNationalPrefixWithCH() throws IOException { + // + is equivalent to 00 in Switzerland + assertTokensInclude(phoneSearchCHAnalyzer, "0041583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testSearchLocalNumberWithCH() throws IOException { + // when omitting the international prefix swiss numbers must start with '0' + assertTokensInclude(phoneSearchCHAnalyzer, "0583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + /** + * Unlike {@link #assertTokenStreamContents(TokenStream, String[])} this only asserts whether the generated tokens + * contain the required ones but does not check for order. Use {@link #assertTokensInclude} if completeness is not needed. + */ + private void assertTokensAreInAnyOrder(final Analyzer analyzer, final String input, final List expectedTokens) + throws IOException { + final var ts = analyzer.tokenStream("test", input); + final var allTokens = getAllTokens(ts).toArray(); + assertThat(allTokens, arrayContainingInAnyOrder(expectedTokens.toArray())); + } + + /** + * Unlike {@link #assertTokenStreamContents(TokenStream, String[])} this only asserts whether the generated tokens + * contain the required ones but does not check for completeness or order. + */ + private void assertTokensInclude(final Analyzer analyzer, final String input, final List expectedTokens) throws IOException { + final var ts = analyzer.tokenStream("test", input); + final var allTokens = getAllTokens(ts).toArray(); + for (final var expectedToken : expectedTokens) { + assertThat(allTokens, hasItemInArray(expectedToken)); + } + } + + /** + * Unlike {@link #assertTokenStreamContents(TokenStream, String[])} this only asserts whether the generated tokens + * contain the required ones but does not check for completeness or order. + * This uses {@link #phoneAnalyzer}. + */ + private void assertTokensInclude(final String input, final List expectedTokens) throws IOException { + this.assertTokensInclude(phoneAnalyzer, input, expectedTokens); + } + + private List getAllTokens(final TokenStream ts) throws IOException { + final var tokens = new ArrayList(); + final var termAtt = ts.getAttribute(CharTermAttribute.class); + ts.reset(); + while (ts.incrementToken()) { + tokens.add(termAtt.toString()); + } + ts.end(); + ts.close(); + return tokens; + } + +} diff --git a/plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json b/plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json new file mode 100644 index 0000000000000..7e45177c57492 --- /dev/null +++ b/plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json @@ -0,0 +1,22 @@ +{ + "index": { + "analysis": { + "analyzer": { + "phone": { + "type": "phone" + }, + "phone-search": { + "type": "phone-search" + }, + "phone-ch": { + "type": "phone", + "phone-region": "CH" + }, + "phone-search-ch": { + "type": "phone-search", + "phone-region": "CH" + } + } + } + } +} diff --git a/plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java b/plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..d514a3329a1a7 --- /dev/null +++ b/plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.analysis.phone; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; +import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; + +public class PhoneNumberAnalysisClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { + public PhoneNumberAnalysisClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return OpenSearchClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml new file mode 100644 index 0000000000000..5bea0cf4650d6 --- /dev/null +++ b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml @@ -0,0 +1,8 @@ +"Test that the plugin is loaded in OpenSearch": + - do: + cat.plugins: + local: true + h: component + + - match: + $body: /^analysis-phonenumber\n$/ diff --git a/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml new file mode 100644 index 0000000000000..0bd7d2c371bfc --- /dev/null +++ b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml @@ -0,0 +1,56 @@ +# Integration tests for phone analysis components +# +--- +"Index phone number content": + - do: + indices.create: + index: test + body: + settings: + index: + analysis: + analyzer: + phone-ch: + type: "phone" + "phone-region": "CH" + phone-search-ch: + type: "phone-search" + "phone-region": "CH" + mappings: + properties: + phone: + type: text + analyzer: "phone" + search_analyzer: "phone-search" + phone-ch: + type: text + analyzer: "phone-ch" + search_analyzer: "phone-search-ch" + + - do: + index: + index: test + id: 1 + body: { "phone": "+41 58 316 10 10", "phone-ch": "058 316 10 10" } + - do: + indices.refresh: {} + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + match: + "phone": "+41583161010" + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + match: + "phone-ch": "+41583161010" + - match: { hits.total: 1 } From f9e0c85c07cf9aa043e714b610d87dcb9bc9b163 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 4 Oct 2024 00:54:50 -0400 Subject: [PATCH 101/108] Fix japicmp configuration by treating abstract-to-default method changes as non-breaking (#16181) Signed-off-by: Andriy Redko --- server/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/build.gradle b/server/build.gradle index 83a04ef12d13b..eccaf8a127647 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -407,6 +407,8 @@ tasks.named("sourcesJar").configure { /** Compares the current build against a laltest released version or the version supplied through 'japicmp.compare.version' system property */ tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) { logger.info("Comparing public APIs from ${version} to ${japicmpCompareTarget}") + // See please https://github.com/siom79/japicmp/issues/201 + compatibilityChangeExcludes = [ "METHOD_ABSTRACT_NOW_DEFAULT" ] oldClasspath.from(files("${buildDir}/japicmp-target/opensearch-${japicmpCompareTarget}.jar")) newClasspath.from(tasks.named('jar')) onlyModified = true From aef7ecace31a4ff3986bb287fec42f3fcfb30bad Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 4 Oct 2024 10:50:53 +0530 Subject: [PATCH 102/108] Add more unit tests for RemoteStoreUtils and RemoteFsTimestampAwareTranslog (#16151) Signed-off-by: Sachin Kale --- .../index/remote/RemoteStoreUtils.java | 1 + .../RemoteFsTimestampAwareTranslog.java | 15 +- .../index/remote/RemoteStoreUtilsTests.java | 97 +++++++ .../RemoteFsTimestampAwareTranslogTests.java | 249 +++++++++++++++++- 4 files changed, 353 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java index edcfe17d31fb8..32a1ca0e5d5ab 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java @@ -477,6 +477,7 @@ private static Set getPinnedTimestampLockedFiles( for (Long pinnedTimestamp : pinnedTimestampSet) { String cachedFile = metadataFilePinnedTimestampMap.get(pinnedTimestamp); if (cachedFile != null) { + assert metadataFiles.contains(cachedFile) : "Metadata files should contain [" + cachedFile + "]"; implicitLockedFiles.add(cachedFile); } else { newPinnedTimestamps.add(pinnedTimestamp); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index ede422ea3c4f7..3ccacde22bbfc 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -233,6 +233,7 @@ public void onResponse(List blobMetadata) { // Update cache to keep only those metadata files that are not getting deleted oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted); + oldFormatMetadataFilePrimaryTermMap.keySet().retainAll(metadataFilesNotToBeDeleted); // Delete stale primary terms deleteStaleRemotePrimaryTerms(metadataFilesNotToBeDeleted); } else { @@ -408,9 +409,9 @@ protected Tuple getMinMaxTranslogGenerationFromMetadataFile( } } - private void deleteStaleRemotePrimaryTerms(List metadataFiles) { + private void deleteStaleRemotePrimaryTerms(List metadataFilesNotToBeDeleted) { deleteStaleRemotePrimaryTerms( - metadataFiles, + metadataFilesNotToBeDeleted, translogTransferManager, oldFormatMetadataFilePrimaryTermMap, minPrimaryTermInRemote, @@ -425,7 +426,7 @@ private void deleteStaleRemotePrimaryTerms(List metadataFiles) { * This will also delete all stale translog metadata files from remote except the latest basis the metadata file comparator. */ protected static void deleteStaleRemotePrimaryTerms( - List metadataFiles, + List metadataFilesNotToBeDeleted, TranslogTransferManager translogTransferManager, Map> oldFormatMetadataFilePrimaryTermMap, AtomicLong minPrimaryTermInRemoteAtomicLong, @@ -434,15 +435,15 @@ protected static void deleteStaleRemotePrimaryTerms( // The deletion of older translog files in remote store is on best-effort basis, there is a possibility that there // are older files that are no longer needed and should be cleaned up. In here, we delete all files that are part // of older primary term. - if (metadataFiles.isEmpty()) { + if (metadataFilesNotToBeDeleted.isEmpty()) { logger.trace("No metadata is uploaded yet, returning from deleteStaleRemotePrimaryTerms"); return; } - Optional minPrimaryTermFromMetadataFiles = metadataFiles.stream().map(file -> { + Optional minPrimaryTermFromMetadataFiles = metadataFilesNotToBeDeleted.stream().map(file -> { try { return getMinMaxPrimaryTermFromMetadataFile(file, translogTransferManager, oldFormatMetadataFilePrimaryTermMap).v1(); } catch (IOException e) { - return Long.MAX_VALUE; + return Long.MIN_VALUE; } }).min(Long::compareTo); // First we delete all stale primary terms folders from remote store @@ -459,7 +460,7 @@ protected static void deleteStaleRemotePrimaryTerms( } } - private static Long getMinPrimaryTermInRemote( + protected static Long getMinPrimaryTermInRemote( AtomicLong minPrimaryTermInRemote, TranslogTransferManager translogTransferManager, Logger logger diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java index be30de97ee830..d134a2bad7b74 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java @@ -1083,4 +1083,101 @@ public void testIsPinnedTimestampStateStaleFeatureEnabled() { setupRemotePinnedTimestampFeature(true); assertTrue(RemoteStoreUtils.isPinnedTimestampStateStale()); } + + public void testGetPinnedTimestampLockedFilesWithCache() { + setupRemotePinnedTimestampFeature(true); + + Map metadataFilePinnedTimestampCache = new HashMap<>(); + + // Pinned timestamps 800, 900, 1000, 2000 + // Metadata with timestamp 990, 995, 1000, 1001 + // Metadata timestamp 1000 <= Pinned Timestamp 1000 + // Metadata timestamp 1001 <= Pinned Timestamp 2000 + Tuple, Set> metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(990L, 995L, 1000L, 1001L), + Set.of(800L, 900L, 1000L, 2000L), + metadataFilePinnedTimestampCache + ); + Map metadataFiles = metadataAndLocks.v1(); + Set implicitLockedFiles = metadataAndLocks.v2(); + + assertEquals(2, implicitLockedFiles.size()); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(1000L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(1001L))); + // Now we cache all the matches except the last one. + assertEquals(1, metadataFilePinnedTimestampCache.size()); + assertEquals(metadataFiles.get(1000L), metadataFilePinnedTimestampCache.get(1000L)); + + metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(990L, 995L, 1000L, 1001L, 2000L, 2200L), + Set.of(800L, 900L, 1000L, 2000L, 3000L), + metadataFilePinnedTimestampCache + ); + metadataFiles = metadataAndLocks.v1(); + implicitLockedFiles = metadataAndLocks.v2(); + assertEquals(3, implicitLockedFiles.size()); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(1000L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(2000L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(2200L))); + assertEquals(2, metadataFilePinnedTimestampCache.size()); + assertEquals(metadataFiles.get(1000L), metadataFilePinnedTimestampCache.get(1000L)); + assertEquals(metadataFiles.get(2000L), metadataFilePinnedTimestampCache.get(2000L)); + + metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(990L, 995L, 1000L, 1001L, 2000L, 2200L, 2500L), + Set.of(2000L, 3000L), + metadataFilePinnedTimestampCache + ); + metadataFiles = metadataAndLocks.v1(); + implicitLockedFiles = metadataAndLocks.v2(); + assertEquals(2, implicitLockedFiles.size()); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(2000L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(2500L))); + assertEquals(1, metadataFilePinnedTimestampCache.size()); + assertEquals(metadataFiles.get(2000L), metadataFilePinnedTimestampCache.get(2000L)); + + metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(2000L, 2200L, 2500L, 3001L, 4200L, 4600L, 5010L), + Set.of(3000L, 4000L, 5000L, 6000L), + metadataFilePinnedTimestampCache + ); + metadataFiles = metadataAndLocks.v1(); + implicitLockedFiles = metadataAndLocks.v2(); + assertEquals(4, implicitLockedFiles.size()); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(2500L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(3001L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(4600L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(5010L))); + assertEquals(3, metadataFilePinnedTimestampCache.size()); + assertEquals(metadataFiles.get(2500L), metadataFilePinnedTimestampCache.get(3000L)); + assertEquals(metadataFiles.get(3001L), metadataFilePinnedTimestampCache.get(4000L)); + assertEquals(metadataFiles.get(4600L), metadataFilePinnedTimestampCache.get(5000L)); + + metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(), + Set.of(3000L, 4000L, 5000L, 6000L), + metadataFilePinnedTimestampCache + ); + implicitLockedFiles = metadataAndLocks.v2(); + assertEquals(0, implicitLockedFiles.size()); + assertEquals(3, metadataFilePinnedTimestampCache.size()); + + assertThrows( + AssertionError.class, + () -> testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(2000L, 2200L, 3001L, 4200L, 4600L, 5010L), + Set.of(3000L, 4000L, 5000L, 6000L), + metadataFilePinnedTimestampCache + ) + ); + + metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(2000L, 2200L, 2500L, 3001L, 4200L, 4600L, 5010L), + Set.of(), + metadataFilePinnedTimestampCache + ); + implicitLockedFiles = metadataAndLocks.v2(); + assertEquals(0, implicitLockedFiles.size()); + assertEquals(0, metadataFilePinnedTimestampCache.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index e6871414cf5e0..73db3314f4d1e 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -721,6 +721,42 @@ public void testGetGenerationsToBeDeleted() throws IOException { assertTrue(generations.isEmpty()); } + public void testGetGenerationsToBeDeletedWithGenerationInRemote() throws IOException { + List metadataFilesNotToBeDeleted = List.of( + // 1 to 4 + "metadata__9223372036854775806__9223372036854775803__9223370311919910398__31__9223372036854775806__1__1", + // 26 to 30 + "metadata__9223372036854775806__9223372036854775777__9223370311919910398__31__9223372036854775781__1__1", + // 42 to 100 + "metadata__9223372036854775806__9223372036854775707__9223370311919910403__31__9223372036854775765__1__1" + ); + List metadataFilesToBeDeleted = List.of( + // 4 to 7 + "metadata__9223372036854775806__9223372036854775800__9223370311919910398__31__9223372036854775803__1__1", + // 17 to 37 + "metadata__9223372036854775806__9223372036854775770__9223370311919910398__31__9223372036854775790__1__1", + // 27 to 42 + "metadata__9223372036854775806__9223372036854775765__9223370311919910403__31__9223372036854775780__1__1" + ); + Set generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted( + metadataFilesNotToBeDeleted, + metadataFilesToBeDeleted, + 35 + ); + Set md1Generations = LongStream.rangeClosed(5, 7).boxed().collect(Collectors.toSet()); + Set md2Generations = LongStream.rangeClosed(17, 25).boxed().collect(Collectors.toSet()); + Set md3Generations = LongStream.rangeClosed(31, 34).boxed().collect(Collectors.toSet()); + + assertTrue(generations.containsAll(md1Generations)); + assertTrue(generations.containsAll(md2Generations)); + assertTrue(generations.containsAll(md3Generations)); + + generations.removeAll(md1Generations); + generations.removeAll(md2Generations); + generations.removeAll(md3Generations); + assertTrue(generations.isEmpty()); + } + public void testGetMetadataFilesToBeDeletedNoExclusion() { updatePinnedTimstampTask.run(); @@ -982,6 +1018,16 @@ public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException translogTransferManager ) ); + // Calling it again to check if the details are getting fetched from the cache. + // Number of calls to readMetadata will be able to verify this. + assertEquals( + new Tuple<>(701L, 1008L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1", + translogTransferManager + ) + ); + TranslogTransferMetadata md2 = mock(TranslogTransferMetadata.class); when(md2.getMinTranslogGeneration()).thenReturn(4L); when(md2.getGeneration()).thenReturn(7L); @@ -994,9 +1040,81 @@ public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException translogTransferManager ) ); + // Calling it again to check if the details are getting fetched from the cache. + // Number of calls to readMetadata will be able to verify this. + assertEquals( + new Tuple<>(4L, 7L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", + translogTransferManager + ) + ); + + verify(translogTransferManager, times(1)).readMetadata( + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1" + ); + verify(translogTransferManager, times(1)).readMetadata( + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1" + ); + } + + public void testGetMinMaxPrimaryTermFromMetadataFile() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + + RemoteFsTimestampAwareTranslog translog = (RemoteFsTimestampAwareTranslog) this.translog; + + // Fetch generations directly from the filename + assertEquals( + new Tuple<>(1L, 1008L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036854774799__9223372036854774799__9223370311919910393__31__9223372036854775106__1__1", + translogTransferManager, + new HashMap<>() + ) + ); + assertEquals( + new Tuple<>(4L, 7L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036854775800__9223372036854775800__9223370311919910398__31__9223372036854775803__4__1", + translogTransferManager, + new HashMap<>() + ) + ); + assertEquals( + new Tuple<>(10L, 10L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036854775797__9223372036854775800__9223370311919910398__31__9223372036854775803__10__1", + translogTransferManager, + new HashMap<>() + ) + ); + + // For older md filenames, it needs to read the content + TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class); + when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "1", "23", "1", "34", "2")); + when(translogTransferManager.readMetadata("metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1")) + .thenReturn(md1); + assertEquals( + new Tuple<>(1L, 2L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1", + translogTransferManager, + new HashMap<>() + ) + ); + assertEquals( + new Tuple<>(4L, 7L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", + translogTransferManager, + Map.of("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", new Tuple<>(4L, 7L)) + ) + ); - verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1"); - verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1"); + verify(translogTransferManager).readMetadata("metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1"); + verify(translogTransferManager, times(0)).readMetadata( + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1" + ); } public void testDeleteStaleRemotePrimaryTerms() throws IOException { @@ -1035,6 +1153,96 @@ public void testDeleteStaleRemotePrimaryTerms() throws IOException { verify(translogTransferManager, times(1)).deletePrimaryTermsAsync(anyLong()); } + public void testDeleteStaleRemotePrimaryTermsOldFormat() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + + List metadataFiles = List.of( + // PT 4 to 9 + "metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1", + // PT 2 to 7 + "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1", + // PT 2 to 6 + "metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1" + ); + + Logger staticLogger = LogManager.getLogger(RemoteFsTimestampAwareTranslogTests.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(1L, 2L, 3L, 4L)); + + TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class); + when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "4", "23", "5", "34", "5")); + when(translogTransferManager.readMetadata("metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1")) + .thenReturn(md1); + TranslogTransferMetadata md2 = mock(TranslogTransferMetadata.class); + when(md2.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "3")); + when(translogTransferManager.readMetadata("metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1")) + .thenReturn(md2); + TranslogTransferMetadata md3 = mock(TranslogTransferMetadata.class); + when(md3.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "2")); + when(translogTransferManager.readMetadata("metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1")) + .thenReturn(md3); + + AtomicLong minPrimaryTermInRemote = new AtomicLong(Long.MAX_VALUE); + RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + new HashMap<>(), + minPrimaryTermInRemote, + staticLogger + ); + verify(translogTransferManager).deletePrimaryTermsAsync(2L); + assertEquals(2, minPrimaryTermInRemote.get()); + + RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + new HashMap<>(), + minPrimaryTermInRemote, + staticLogger + ); + // This means there are no new invocations of deletePrimaryTermAsync + verify(translogTransferManager, times(1)).deletePrimaryTermsAsync(anyLong()); + } + + public void testDeleteStaleRemotePrimaryTermsOldFormatException() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + + List metadataFiles = List.of( + // PT 4 to 9 + "metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1", + // PT 2 to 7 + "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1", + // PT 2 to 6 + "metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1" + ); + + Logger staticLogger = LogManager.getLogger(RemoteFsTimestampAwareTranslogTests.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(1L, 2L, 3L, 4L)); + + TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class); + when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "4", "23", "5", "34", "5")); + when(translogTransferManager.readMetadata("metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1")) + .thenReturn(md1); + TranslogTransferMetadata md2 = mock(TranslogTransferMetadata.class); + when(md2.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "3")); + when(translogTransferManager.readMetadata("metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1")) + .thenReturn(md2); + TranslogTransferMetadata md3 = mock(TranslogTransferMetadata.class); + when(md3.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "2")); + // Exception while reading this file + when(translogTransferManager.readMetadata("metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1")) + .thenThrow(new IOException()); + + AtomicLong minPrimaryTermInRemote = new AtomicLong(4); + RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + new HashMap<>(), + minPrimaryTermInRemote, + staticLogger + ); + verify(translogTransferManager, times(0)).deletePrimaryTermsAsync(anyLong()); + } + public void testDeleteStaleRemotePrimaryTermsNoPrimaryTermInRemote() throws IOException { TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); @@ -1087,4 +1295,41 @@ public void testDeleteStaleRemotePrimaryTermsPrimaryTermInRemoteIsBigger() throw assertEquals(2, minPrimaryTermInRemote.get()); } + public void testGetMinPrimaryTermInRemoteAlreadyFetched() { + Long expectedMinPrimaryTerm = 12L; + assertEquals(expectedMinPrimaryTerm, RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(12), null, null)); + } + + public void testGetMinPrimaryTermInRemoteNotFetchedEmpty() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of()); + Long expectedMinPrimaryTerm = Long.MAX_VALUE; + assertEquals( + expectedMinPrimaryTerm, + RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(Long.MAX_VALUE), translogTransferManager, null) + ); + verify(translogTransferManager).listPrimaryTermsInRemote(); + } + + public void testGetMinPrimaryTermInRemoteNotFetchedException() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenThrow(new IOException()); + Long expectedMinPrimaryTerm = Long.MAX_VALUE; + assertEquals( + expectedMinPrimaryTerm, + RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(Long.MAX_VALUE), translogTransferManager, logger) + ); + verify(translogTransferManager).listPrimaryTermsInRemote(); + } + + public void testGetMinPrimaryTermInRemoteNotFetched() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(12L, 23L, 34L, 4L, 89L)); + Long expectedMinPrimaryTerm = 4L; + assertEquals( + expectedMinPrimaryTerm, + RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(Long.MAX_VALUE), translogTransferManager, logger) + ); + verify(translogTransferManager).listPrimaryTermsInRemote(); + } } From ba8f1bec392d5f4301b9e043d06456eefd89e8fd Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:02:10 +0530 Subject: [PATCH 103/108] Run queued operations post v2 operations completion (#16179) Signed-off-by: Gaurav Bafna --- .../snapshots/ConcurrentSnapshotsV2IT.java | 118 ++++++++++++++++++ .../snapshots/SnapshotsService.java | 24 ++-- 2 files changed, 133 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java index f20fddb6af26c..97d63d0e2380a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java @@ -479,6 +479,124 @@ public void testCloneSnapshotV2MasterSwitch() throws Exception { assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), csr2.getSnapshotInfo())); } + public void testDeleteWhileV2CreateOngoing() throws Exception { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + startFullSnapshot(repoName, "snapshot-v1").actionGet(); + + // Creating a v2 repo + settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + blockClusterManagerOnWriteIndexFile(repoName); + + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-v2"); + awaitNumberOfSnapshotsInProgress(1); + + ActionFuture a = startDeleteSnapshot(repoName, "snapshot-v1"); + + unblockNode(repoName, clusterManagerName); + CreateSnapshotResponse csr = snapshotFuture.actionGet(); + assertTrue(csr.getSnapshotInfo().getPinnedTimestamp() != 0); + assertTrue(a.actionGet().isAcknowledged()); + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(1, snapInfo.size()); + assertThat(snapInfo, contains(csr.getSnapshotInfo())); + } + + public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + startFullSnapshot(repoName, "snapshot-v1").actionGet(); + startFullSnapshot(repoName, "snapshot-v1-2").actionGet(); + + // Creating a v2 repo + settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + blockClusterManagerOnWriteIndexFile(repoName); + + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-v2"); + awaitNumberOfSnapshotsInProgress(1); + + ActionFuture startDeleteSnapshot = startDeleteSnapshot(repoName, "snapshot-v1"); + ActionFuture startCloneSnapshot = startCloneSnapshot(repoName, "snapshot-v1-2", "snapshot-v1-2-clone"); + + unblockNode(repoName, clusterManagerName); + CreateSnapshotResponse csr = snapshotFuture.actionGet(); + assertTrue(csr.getSnapshotInfo().getPinnedTimestamp() != 0); + assertTrue(startDeleteSnapshot.actionGet().isAcknowledged()); + assertTrue(startCloneSnapshot.actionGet().isAcknowledged()); + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(3, snapInfo.size()); + } + protected ActionFuture startCloneSnapshot(String repoName, String sourceSnapshotName, String snapshotName) { logger.info("--> creating full snapshot [{}] to repo [{}]", snapshotName, repoName); return clusterAdmin().prepareCloneSnapshot(repoName, sourceSnapshotName, snapshotName).setIndices("*").execute(); diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index c80f18cdd82f7..541a87c200883 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -382,7 +382,7 @@ public ClusterState execute(ClusterState currentState) { ); if (request.partial() == false) { Set missing = new HashSet<>(); - for (final Map.Entry entry : shards.entrySet()) { + for (final Map.Entry entry : shards.entrySet()) { if (entry.getValue().state() == ShardState.MISSING) { missing.add(entry.getKey().getIndex().getName()); } @@ -606,8 +606,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl new ActionListener() { @Override public void onResponse(RepositoryData repositoryData) { - leaveRepoLoop(repositoryName); if (clusterService.state().nodes().isLocalNodeElectedClusterManager() == false) { + leaveRepoLoop(repositoryName); failSnapshotCompletionListeners( snapshot, new SnapshotException(snapshot, "Aborting snapshot-v2, no longer cluster manager") @@ -622,6 +622,9 @@ public void onResponse(RepositoryData repositoryData) { return; } listener.onResponse(snapshotInfo); + // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations + // can get queued . This is triggering them. + runNextQueuedOperation(repositoryData, repositoryName, true); cleanOrphanTimestamp(repositoryName, repositoryData); } @@ -1010,8 +1013,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl new ActionListener() { @Override public void onResponse(RepositoryData repositoryData) { - leaveRepoLoop(repositoryName); if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { + leaveRepoLoop(repositoryName); failSnapshotCompletionListeners( snapshot, new SnapshotException(snapshot, "Aborting Snapshot-v2 clone, no longer cluster manager") @@ -1027,6 +1030,9 @@ public void onResponse(RepositoryData repositoryData) { } logger.info("snapshot-v2 clone [{}] completed successfully", snapshot); listener.onResponse(null); + // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations + // can get queued . This is triggering them. + runNextQueuedOperation(repositoryData, repositoryName, true); } @Override @@ -1740,7 +1746,7 @@ private static boolean assertNoDanglingSnapshots(ClusterState state) { /** * Updates the state of in-progress snapshots in reaction to a change in the configuration of the cluster nodes (cluster-manager fail-over or * disconnect of a data node that was executing a snapshot) or a routing change that started shards whose snapshot state is - * {@link SnapshotsInProgress.ShardState#WAITING}. + * {@link ShardState#WAITING}. * * @param changedNodes true iff either a cluster-manager fail-over occurred or a data node that was doing snapshot work got removed from the * cluster @@ -2645,7 +2651,7 @@ private static List matchingSnapshotIds( } } } - return Collections.unmodifiableList(new ArrayList<>(foundSnapshots)); + return unmodifiableList(new ArrayList<>(foundSnapshots)); } // Return in-progress snapshot entries by name and repository in the given cluster state or null if none is found @@ -2795,7 +2801,7 @@ public ClusterState execute(ClusterState currentState) { reusedExistingDelete = true; return currentState; } - final List toDelete = Collections.unmodifiableList(new ArrayList<>(snapshotIdsRequiringCleanup)); + final List toDelete = unmodifiableList(new ArrayList<>(snapshotIdsRequiringCleanup)); ensureBelowConcurrencyLimit(repoName, toDelete.get(0).getName(), snapshots, deletionsInProgress); newDelete = new SnapshotDeletionsInProgress.Entry( toDelete, @@ -3353,7 +3359,7 @@ private static void completeListenersIgnoringException(@Nullable List shards( + private static Map shards( SnapshotsInProgress snapshotsInProgress, @Nullable SnapshotDeletionsInProgress deletionsInProgress, Metadata metadata, @@ -3362,7 +3368,7 @@ private static Map shards( RepositoryData repositoryData, String repoName ) { - final Map builder = new HashMap<>(); + final Map builder = new HashMap<>(); final ShardGenerations shardGenerations = repositoryData.shardGenerations(); final InFlightShardSnapshotStates inFlightShardStates = InFlightShardSnapshotStates.forRepo( repoName, @@ -3396,7 +3402,7 @@ private static Map shards( } final ShardSnapshotStatus shardSnapshotStatus; if (indexRoutingTable == null) { - shardSnapshotStatus = new SnapshotsInProgress.ShardSnapshotStatus( + shardSnapshotStatus = new ShardSnapshotStatus( null, ShardState.MISSING, "missing routing table", From b06ddb617c09f65a7804c483d3e3dcc7159c4a29 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 4 Oct 2024 12:48:14 -0400 Subject: [PATCH 104/108] Fix warnings from SLF4J on startup when repository-s3 is installed (#16194) * Fix warnings from SLF4J on startup when repository-s3 is installed Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Fix precommit Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + plugins/repository-s3/build.gradle | 7 ++++--- .../licenses/log4j-slf4j-impl-2.21.0.jar.sha1 | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index ca23cd59e06d5..1a289a8c62000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) - Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521)) - Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158)) +- Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194)) ### Security diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 00decbe4fa9cd..22aa151c92003 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -82,6 +82,8 @@ dependencies { api "joda-time:joda-time:${versions.joda}" api "org.slf4j:slf4j-api:${versions.slf4j}" + runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" + // network stack api "io.netty:netty-buffer:${versions.netty}" api "io.netty:netty-codec:${versions.netty}" @@ -111,6 +113,7 @@ tasks.named("dependencyLicenses").configure { mapping from: /jackson-.*/, to: 'jackson' mapping from: /jaxb-.*/, to: 'jaxb' mapping from: /netty-.*/, to: 'netty' + mapping from: /log4j-.*/, to: 'log4j' } bundlePlugin { @@ -510,9 +513,7 @@ thirdPartyAudit { 'org.jboss.marshalling.MarshallingConfiguration', 'org.jboss.marshalling.Unmarshaller', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', + 'org.slf4j.ext.EventData', 'reactor.blockhound.BlockHound$Builder', 'reactor.blockhound.integration.BlockHoundIntegration', diff --git a/plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1 b/plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1 new file mode 100644 index 0000000000000..0e22f98daa61c --- /dev/null +++ b/plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1 @@ -0,0 +1 @@ +911fdb5b1a1df36719c579ecc6f2957b88bce1ab \ No newline at end of file From 421a1cc2270ef02f02fc7621895f8b15ec7282f1 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 4 Oct 2024 22:24:36 +0530 Subject: [PATCH 105/108] Fix unknown parameter source_remote_translog_repository bug (#16192) Signed-off-by: Sachin Kale --- .../RestoreShallowSnapshotV2IT.java | 83 +++++++++++++++++++ .../restore/RestoreSnapshotRequest.java | 6 ++ .../RestoreSnapshotRequestBuilder.java | 8 ++ 3 files changed, 97 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java index c5a55f16cab2b..24f1141ddbede 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java @@ -10,6 +10,8 @@ import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; @@ -19,6 +21,7 @@ import org.opensearch.client.Requests; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobPath; @@ -64,6 +67,7 @@ import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -106,6 +110,18 @@ protected Settings.Builder getRepositorySettings(Path location, boolean shallowC return settingsBuilder; } + protected Settings.Builder getRepositorySettings(String sourceRepository, boolean readOnly) throws ExecutionException, + InterruptedException { + GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { sourceRepository }); + GetRepositoriesResponse res = client().admin().cluster().getRepositories(gr).get(); + RepositoryMetadata rmd = res.repositories().get(0); + return Settings.builder() + .put(rmd.settings()) + .put(BlobStoreRepository.READONLY_SETTING.getKey(), readOnly) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false) + .put(SYSTEM_REPOSITORY_SETTING.getKey(), false); + } + private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { Settings.Builder settingsBuilder = Settings.builder() .put(super.indexSettings()) @@ -802,4 +818,71 @@ public void testInvalidRestoreRequestScenarios() throws Exception { ); assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore")); } + + public void testRestoreOperationsUsingDifferentRepos() throws Exception { + disableRepoConsistencyCheck("Remote store repo"); + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + String primary = internalCluster().startDataOnlyNode(); + String indexName1 = "testindex1"; + String snapshotRepoName = "test-snapshot-repo"; + String snapshotName1 = "test-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + // Create repo + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true)); + + // Create index + Client client = client(); + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + ensureGreen(indexName1); + + // Index 5 documents, refresh, index 5 documents + final int numDocsInIndex1 = 5; + indexDocuments(client, indexName1, 0, numDocsInIndex1); + refresh(indexName1); + indexDocuments(client, indexName1, numDocsInIndex1, 2 * numDocsInIndex1); + + // Take V2 snapshot + logger.info("--> snapshot"); + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>()); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + + // Create new snapshot, segment and translog repositories + String newSnapshotRepo = "backup-snapshot"; + String newSegmentRepo = "backup-segment"; + String newTranslogRepo = "backup-translog"; + createRepository(newSnapshotRepo, "fs", getRepositorySettings(snapshotRepoName, true)); + createRepository(newSegmentRepo, "fs", getRepositorySettings(BASE_REMOTE_REPO, true)); + createRepository(newTranslogRepo, "fs", getRepositorySettings(BASE_REMOTE_REPO, true)); + + // Delete index + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName1)).get()); + assertFalse(indexExists(indexName1)); + + // Restore using new repos + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(newSnapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndices(indexName1) + .setSourceRemoteStoreRepository(newSegmentRepo) + .setSourceRemoteTranslogRepository(newTranslogRepo) + .get(); + + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + + // Verify restored index's stats + ensureYellowAndNoInitializingShards(indexName1); + ensureGreen(indexName1); + assertDocsPresentInIndex(client(), indexName1, 2 * numDocsInIndex1); + + // indexing some new docs and validating + indexDocuments(client, indexName1, 2 * numDocsInIndex1, 3 * numDocsInIndex1); + ensureGreen(indexName1); + assertDocsPresentInIndex(client, indexName1, 3 * numDocsInIndex1); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 469a31407dc5d..f3110cc8f20a5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -652,6 +652,12 @@ public RestoreSnapshotRequest source(Map source) { } else { throw new IllegalArgumentException("malformed source_remote_store_repository"); } + } else if (name.equals("source_remote_translog_repository")) { + if (entry.getValue() instanceof String) { + setSourceRemoteTranslogRepository((String) entry.getValue()); + } else { + throw new IllegalArgumentException("malformed source_remote_translog_repository"); + } } else { if (IndicesOptions.isIndicesOptions(name) == false) { throw new IllegalArgumentException("Unknown parameter " + name); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index 39eaadf3c8de6..53c9557a621b7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -266,4 +266,12 @@ public RestoreSnapshotRequestBuilder setSourceRemoteStoreRepository(String repos request.setSourceRemoteStoreRepository(repositoryName); return this; } + + /** + * Sets the source remote translog repository name + */ + public RestoreSnapshotRequestBuilder setSourceRemoteTranslogRepository(String repositoryName) { + request.setSourceRemoteTranslogRepository(repositoryName); + return this; + } } From 848234e16ab1ffc8a8a71b15f0e60a8512e04e95 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 4 Oct 2024 12:56:53 -0400 Subject: [PATCH 106/108] Modifies sharedwith to accomodate scope Signed-off-by: Darshit Chanpura --- .../accesscontrol/resources/ShareWith.java | 59 ++++------- .../resources/SharedWithScope.java | 98 +++++++++++++++++++ 2 files changed, 118 insertions(+), 39 deletions(-) create mode 100644 server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java index 8bb3730bacd19..df63e8fa8f02d 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java @@ -15,56 +15,37 @@ import java.util.List; /** - * This class contains information about whom a resource is shared with. - * It could be a user-name, a role or a backend_role. + * This class contains information about whom a resource is shared with and at what scope. + * Here is a sample of what this would look like: + * "share_with": { + * "read_only": { + * "users": [], + * "roles": [], + * "backend_roles": [] + * }, + * "read_write": { + * "users": [], + * "roles": [], + * "backend_roles": [] + * } + * } * * @opensearch.experimental */ public class ShareWith implements ToXContentFragment { - private List users; + private final List sharedWithScopes; - private List roles; - - private List backendRoles; - - public ShareWith(List users, List roles, List backendRoles) { - this.users = users; - this.roles = roles; - this.backendRoles = backendRoles; - } - - public List getUsers() { - return users; - } - - public void setUsers(List users) { - this.users = users; - } - - public List getRoles() { - return roles; + public ShareWith(List sharedWithScopes) { + this.sharedWithScopes = sharedWithScopes; } - public void setRoles(List roles) { - this.roles = roles; - } - - public List getBackendRoles() { - return backendRoles; - } - - public void setBackendRoles(List backendRoles) { - this.backendRoles = backendRoles; - } - - @Override - public String toString() { - return "ShareWith {" + "users=" + users + ", roles=" + roles + ", backendRoles=" + backendRoles + '}'; + public List getSharedWithScopes() { + return sharedWithScopes; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject().field("users", users).field("roles", roles).field("backend_roles", backendRoles).endObject(); + return builder.startObject("share_with").value(sharedWithScopes).endObject(); } } diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java b/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java new file mode 100644 index 0000000000000..2bc1f4614e227 --- /dev/null +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java @@ -0,0 +1,98 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.accesscontrol.resources; + +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; + +/** + * Defines the scope and who this scope is shared with + * + * @opensearch.experimental + */ +public class SharedWithScope implements ToXContentFragment { + + private final String scope; + + private final SharedWithPerScope sharedWithPerScope; + + public SharedWithScope(String scope, SharedWithPerScope sharedWithPerScope) { + this.scope = scope; + this.sharedWithPerScope = sharedWithPerScope; + } + + public String getScope() { + return scope; + } + + public SharedWithPerScope getSharedWithPerScope() { + return sharedWithPerScope; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject().field(scope, sharedWithPerScope).endObject(); + } + + @Override + public String toString() { + return "SharedWithScope {" + scope + ": " + sharedWithPerScope + '}'; + } + + public static class SharedWithPerScope implements ToXContentFragment { + private List users; + + private List roles; + + private List backendRoles; + + public SharedWithPerScope(List users, List roles, List backendRoles) { + this.users = users; + this.roles = roles; + this.backendRoles = backendRoles; + } + + public List getUsers() { + return users; + } + + public void setUsers(List users) { + this.users = users; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public List getBackendRoles() { + return backendRoles; + } + + public void setBackendRoles(List backendRoles) { + this.backendRoles = backendRoles; + } + + @Override + public String toString() { + return "ShareWith {" + "users=" + users + ", roles=" + roles + ", backendRoles=" + backendRoles + '}'; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject().field("users", users).field("roles", roles).field("backend_roles", backendRoles).endObject(); + } + } +} From eaf0c6ea96075508a71c8793bcf6c16ad4468f7a Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 4 Oct 2024 13:27:04 -0400 Subject: [PATCH 107/108] Adds missing JavaDoc Signed-off-by: Darshit Chanpura --- .../opensearch/accesscontrol/resources/SharedWithScope.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java b/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java index 2bc1f4614e227..744419ad4a7be 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java @@ -48,6 +48,11 @@ public String toString() { return "SharedWithScope {" + scope + ": " + sharedWithPerScope + '}'; } + /** + * This class defines who a resource is shared_with for a particular scope + * + * @opensearch.experimental + */ public static class SharedWithPerScope implements ToXContentFragment { private List users; From 566913ae6ff7009d3c3bda9a2464934263f3f80e Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 4 Oct 2024 18:14:15 -0400 Subject: [PATCH 108/108] Adds NamedWriteable capability and removes un-needed method Signed-off-by: Darshit Chanpura --- .../resources/ResourceService.java | 2 +- .../accesscontrol/resources/ShareWith.java | 14 ++++++++- .../resources/SharedWithScope.java | 29 +++++++++++++++++-- .../NoOpResourceAccessControlPlugin.java | 11 +------ .../plugins/ResourceAccessControlPlugin.java | 7 +---- 5 files changed, 43 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java index 1a5fd9c8d8701..41ccd4739d7ec 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java @@ -56,7 +56,7 @@ public ResourceAccessControlPlugin getResourceAccessControlPlugin() { /** * List active plugins that define resources */ - public List listResourcePlugins() { + List listResourcePlugins() { return resourcePlugins; } } diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java index df63e8fa8f02d..2018cdfdfd96d 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java @@ -8,6 +8,8 @@ package org.opensearch.accesscontrol.resources; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -32,7 +34,7 @@ * * @opensearch.experimental */ -public class ShareWith implements ToXContentFragment { +public class ShareWith implements ToXContentFragment, NamedWriteable { private final List sharedWithScopes; @@ -48,4 +50,14 @@ public List getSharedWithScopes() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.startObject("share_with").value(sharedWithScopes).endObject(); } + + @Override + public String getWriteableName() { + return "share_with"; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeList(sharedWithScopes); + } } diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java b/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java index 744419ad4a7be..1f424177fcd4e 100644 --- a/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java +++ b/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java @@ -8,6 +8,8 @@ package org.opensearch.accesscontrol.resources; +import org.opensearch.core.common.io.stream.NamedWriteable; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -19,7 +21,7 @@ * * @opensearch.experimental */ -public class SharedWithScope implements ToXContentFragment { +public class SharedWithScope implements ToXContentFragment, NamedWriteable { private final String scope; @@ -48,12 +50,23 @@ public String toString() { return "SharedWithScope {" + scope + ": " + sharedWithPerScope + '}'; } + @Override + public String getWriteableName() { + return "shared_with_scope"; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(scope); + out.writeNamedWriteable(sharedWithPerScope); + } + /** * This class defines who a resource is shared_with for a particular scope * * @opensearch.experimental */ - public static class SharedWithPerScope implements ToXContentFragment { + public static class SharedWithPerScope implements ToXContentFragment, NamedWriteable { private List users; private List roles; @@ -99,5 +112,17 @@ public String toString() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.startObject().field("users", users).field("roles", roles).field("backend_roles", backendRoles).endObject(); } + + @Override + public String getWriteableName() { + return "shared_with_per_scope"; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringArray(users.toArray(new String[0])); + out.writeStringArray(roles.toArray(new String[0])); + out.writeStringArray(backendRoles.toArray(new String[0])); + } } } diff --git a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java index 2619b05124b67..ad67ecc6e9ef1 100644 --- a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java @@ -22,15 +22,6 @@ */ public class NoOpResourceAccessControlPlugin implements ResourceAccessControlPlugin { - /** - * - * @return an empty map of resource names accessible by this user. - */ - @Override - public Map> listAccessibleResources() { - return Map.of(); - } - /** * Returns an empty list since security plugin is not defined. * This method alone doesn't determine permissions. @@ -38,7 +29,7 @@ public Map> listAccessibleResources() { * @return empty list */ @Override - public List listAccessibleResourcesForPlugin(String systemIndexName) { + public List listAccessibleResourcesInPlugin(String systemIndexName) { // returns an empty list since security plugin is disabled return List.of(); } diff --git a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java index 8298a65becfd7..379fee4be090b 100644 --- a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java @@ -27,18 +27,13 @@ * @opensearch.experimental */ public interface ResourceAccessControlPlugin { - /** - * Returns all accessible resources for current user. - * @return a map of resources accessible by user separated by system index names - */ - Map> listAccessibleResources(); /** * Returns all accessible resources for current user for a given system . * * @return list of {@link ResourceSharing} items accessible by current user. */ - List listAccessibleResourcesForPlugin(String systemIndex); + List listAccessibleResourcesInPlugin(String systemIndex); /** * Checks whether current user has permission to given resource.