diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index 0f59fec3d824a..5587b8e5784c5 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -38,4 +38,6 @@ BWC_VERSION:
   - "2.16.0"
   - "2.16.1"
   - "2.17.0"
+  - "2.17.1"
+  - "2.17.2"
   - "2.18.0"
diff --git a/.github/benchmark-configs.json b/.github/benchmark-configs.json
index 4ada715d21495..27b7228e1203a 100644
--- a/.github/benchmark-configs.json
+++ b/.github/benchmark-configs.json
@@ -75,7 +75,7 @@
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -126,7 +126,7 @@
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -176,7 +176,7 @@
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "search.concurrent_segment_search.enabled:true",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -194,7 +194,7 @@
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:all",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -212,7 +212,7 @@
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:auto",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 8c4f4d59ea1fc..af83f8b2edac2 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -722,6 +722,12 @@ updates:
   - directory: /plugins/
     open-pull-requests-limit: 1
     package-ecosystem: gradle
+    ignore:
+      # For all packages, ignore all major versions to minimize breaking issues
+      - dependency-name: "com.google.cloud:google-cloud-storage"
+        update-types: [ "version-update:semver-major" ]
+      - dependency-name: "com.google.api-client:google-api-client"
+        update-types: [ "version-update:semver-major" ]
     schedule:
       interval: weekly
     labels:
diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml
index 387a124b8cb6a..1c73d3d250ce3 100644
--- a/.github/workflows/delete_backport_branch.yml
+++ b/.github/workflows/delete_backport_branch.yml
@@ -7,9 +7,16 @@ on:
 jobs:
   delete-branch:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     if: startsWith(github.event.pull_request.head.ref,'backport/')
     steps:
-      - name: Delete merged branch
-        uses: SvanBoxel/delete-merged-branch@main
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Delete merged branch
+      uses: actions/github-script@v7
+      with:
+        script: |
+          github.rest.git.deleteRef({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            ref: `heads/${context.payload.pull_request.head.ref}`,
+          })
diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml
index 1b9b30625eb83..1421eeb7f7576 100644
--- a/.github/workflows/gradle-check.yml
+++ b/.github/workflows/gradle-check.yml
@@ -8,6 +8,10 @@ on:
   pull_request_target:
     types: [opened, synchronize, reopened]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
 permissions:
   contents: read # to fetch code (actions/checkout)
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3268852cc99f9..ec42f4902294f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,18 +5,45 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621))
 - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637))
 - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651))
 - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424))
+- Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916))
+- Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967))
+- Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923))
+- Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718))
+- Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976))
+- Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611))
+- [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978))
+- Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986))
+- New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915))
+- Add resource-level access control and sharing ([#16030](https://github.com/opensearch-project/OpenSearch/pull/16030))
 
 ### Dependencies
 - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578))
 - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684))
-- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.1 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858), [#16134](https://github.com/opensearch-project/OpenSearch/pull/16134))
 - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863))
 - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862))
+- Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945))
+- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946))
+- Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011))
+- Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040))
+- Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038))
+- Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039))
+- Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041))
+- Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042))
+- Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137))
+- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133))
+- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135))
+- Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182))
 
 ### Changed
+- Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049))
+- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430))
+- Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024))
+- Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154))
 
 
 ### Deprecated
@@ -25,6 +52,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Fixed
 - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737))
+- Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882))
+- Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501))
+- Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988))
+- Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985))
+- Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931))
+- Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521))
+- Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158))
+- Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194))
 
 ### Security
 
diff --git a/README.md b/README.md
index 5d4a9a671c013..95fbac7bbecf1 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,8 @@
 [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
 [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
 [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
-[![2.17.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.0")
+[![2.18.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0")
+[![2.17.2 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.2)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.2")
 [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
 [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
 [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)
diff --git a/TESTING.md b/TESTING.md
index de7ab3eefe2f8..9127f005ced75 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -39,7 +39,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran
     - [Expect a specific segment topology](#expect-a-specific-segment-topology)
     - [Leave environment in an unstable state after test](#leave-environment-in-an-unstable-state-after-test)
 - [Test coverage analysis](#test-coverage-analysis)
-- [Building with extra plugins](#building-with-extra-plugins)
+- [Testing with plugins](#testing-with-plugins)
 - [Environment misc](#environment-misc)
 
 # Requirements
@@ -552,11 +552,17 @@ Apart from using Gradle, it is also possible to gain insight in code coverage us
 
 Please read your IDE documentation for how to attach a debugger to a JVM process.
 
-# Building with extra plugins
+# Testing with plugins
 
-Additional plugins may be built alongside OpenSearch, where their dependency on OpenSearch will be substituted with the local OpenSearch build. To add your plugin, create a directory called `opensearch-extra` as a sibling of OpenSearch. Checkout your plugin underneath `opensearch-extra` and the build will automatically pick it up. You can verify the plugin is included as part of the build by checking the projects of the build.
+To test a plugin with a custom build of OpenSearch, build OpenSearch and use the `customDistributionUrl` setting supported by each plugin to override the OpenSearch distribution.
 
-    ./gradlew projects
+For example, in your OpenSearch repository assemble a custom distribution.
+
+    ./gradlew :distribution:archives:linux-tar:assemble
+
+Then in your plugin repository, substitute in your OpenSearch build
+
+    ./gradlew run -PcustomDistributionUrl="<OPENSEARCH-REPO-PATH>/distribution/archives/linux-tar/build/distributions/opensearch-min-3.0.0-SNAPSHOT-linux-x64.tar.gz"
 
 # Environment misc
 
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java
index fc78792bb3551..77d7997d6d48d 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java
@@ -106,6 +106,7 @@ public DockerAvailability getDockerAvailability() {
             Version version = null;
             boolean isVersionHighEnough = false;
             boolean isComposeAvailable = false;
+            boolean isComposeV2Available = false;
 
             // Check if the Docker binary exists
             final Optional<String> dockerBinary = getDockerPath();
@@ -129,6 +130,8 @@ public DockerAvailability getDockerAvailability() {
                         if (lastResult.isSuccess() && composePath.isPresent()) {
                             isComposeAvailable = runCommand(composePath.get(), "version").isSuccess();
                         }
+
+                        isComposeV2Available = runCommand(dockerPath, "compose", "version").isSuccess();
                     }
                 }
             }
@@ -138,6 +141,7 @@ public DockerAvailability getDockerAvailability() {
             this.dockerAvailability = new DockerAvailability(
                 isAvailable,
                 isComposeAvailable,
+                isComposeV2Available,
                 isVersionHighEnough,
                 dockerPath,
                 version,
@@ -356,6 +360,11 @@ public static class DockerAvailability {
          */
         public final boolean isComposeAvailable;
 
+        /**
+         * True if docker compose is available.
+         */
+        public final boolean isComposeV2Available;
+
         /**
          * True if the installed Docker version is &gt;= 17.05
          */
@@ -379,6 +388,7 @@ public static class DockerAvailability {
         DockerAvailability(
             boolean isAvailable,
             boolean isComposeAvailable,
+            boolean isComposeV2Available,
             boolean isVersionHighEnough,
             String path,
             Version version,
@@ -386,6 +396,7 @@ public static class DockerAvailability {
         ) {
             this.isAvailable = isAvailable;
             this.isComposeAvailable = isComposeAvailable;
+            this.isComposeV2Available = isComposeV2Available;
             this.isVersionHighEnough = isVersionHighEnough;
             this.path = path;
             this.version = version;
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java
index 268de50340cbf..cd22560af9a96 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java
@@ -116,7 +116,12 @@ public class OpenSearchNode implements TestClusterConfiguration {
     private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.MINUTES;
     private static final int ADDITIONAL_CONFIG_TIMEOUT = 15;
     private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS;
-    private static final List<String> OVERRIDABLE_SETTINGS = Arrays.asList("path.repo", "discovery.seed_providers", "discovery.seed_hosts");
+    private static final List<String> OVERRIDABLE_SETTINGS = Arrays.asList(
+        "path.repo",
+        "discovery.seed_providers",
+        "discovery.seed_hosts",
+        "indices.breaker.total.use_real_memory"
+    );
 
     private static final int TAIL_LOG_MESSAGES_COUNT = 40;
     private static final List<String> MESSAGES_WE_DONT_CARE_ABOUT = Arrays.asList(
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java
index e8772522b19a4..f65e231cd2e50 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java
@@ -171,7 +171,11 @@ public void execute(Task task) {
                 .findFirst();
 
             composeExtension.getExecutable().set(dockerCompose.isPresent() ? dockerCompose.get() : "/usr/bin/docker");
-            composeExtension.getUseDockerComposeV2().set(false);
+            if (dockerSupport.get().getDockerAvailability().isComposeV2Available) {
+                composeExtension.getUseDockerComposeV2().set(true);
+            } else if (dockerSupport.get().getDockerAvailability().isComposeAvailable) {
+                composeExtension.getUseDockerComposeV2().set(false);
+            }
 
             tasks.named("composeUp").configure(t -> {
                 // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions
@@ -228,7 +232,8 @@ private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, TaskPro
 
     private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, Task task) {
         task.onlyIf(spec -> {
-            boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeAvailable;
+            boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeV2Available
+                || dockerSupport.get().getDockerAvailability().isComposeAvailable;
             if (isComposeAvailable == false) {
                 LOGGER.info("Task {} requires docker-compose but it is unavailable. Task will be skipped.", task.getPath());
             }
diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
index 59063b3789c70..4d425964c77af 100644
--- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
+++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
@@ -17,7 +17,7 @@ repositories {
 }
 
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.24.0"
+  implementation "org.apache.logging.log4j:log4j-core:2.24.1"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 0c6d0536dd1d5..8cede6b717883 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -20,7 +20,7 @@ woodstox          = 6.4.0
 kotlin            = 1.7.10
 antlr4            = 4.13.1
 guava             = 32.1.1-jre
-protobuf          = 3.25.4
+protobuf          = 3.25.5
 jakarta_annotation = 1.3.5
 google_http_client = 1.44.1
 tdigest           = 3.3
@@ -29,7 +29,7 @@ hdrhistogram      = 2.2.2
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.112.Final
+netty             = 4.1.114.Final
 joda              = 2.12.7
 
 # project reactor
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java
index b0a7d1e3578c0..b512117c42f65 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java
@@ -90,6 +90,7 @@
 public abstract class OpenSearchRestHighLevelClientTestCase extends OpenSearchRestTestCase {
 
     protected static final String CONFLICT_PIPELINE_ID = "conflict_pipeline";
+    protected static final double DOUBLE_DELTA = 0.000001;
 
     private static RestHighLevelClient restHighLevelClient;
     private static boolean async = Booleans.parseBoolean(System.getProperty("tests.rest.async", "false"));
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java
index 01fdd489aa7d8..6da8a29a9789e 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java
@@ -158,7 +158,7 @@ public void testMetrics() throws IOException {
 
             RankEvalRequest rankEvalRequest = new RankEvalRequest(spec, new String[] { "index", "index2" });
             RankEvalResponse response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
-            assertEquals(expectedScores[i], response.getMetricScore(), Double.MIN_VALUE);
+            assertEquals(expectedScores[i], response.getMetricScore(), DOUBLE_DELTA);
             i++;
         }
     }
diff --git a/gradle/missing-javadoc.gradle b/gradle/missing-javadoc.gradle
index e9a6d798b8323..26898673bf608 100644
--- a/gradle/missing-javadoc.gradle
+++ b/gradle/missing-javadoc.gradle
@@ -127,6 +127,7 @@ configure([
   project(":plugins:analysis-icu"),
   project(":plugins:analysis-kuromoji"),
   project(":plugins:analysis-nori"),
+  project(":plugins:analysis-phonenumber"),
   project(":plugins:analysis-phonetic"),
   project(":plugins:analysis-smartcn"),
   project(":plugins:analysis-stempel"),
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 5c6205ebf24d4..e312a2da77d94 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15
+distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index ca5dd306cf907..5d38f85c40c89 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -109,6 +109,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_16_1 = new Version(2160199, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1);
+    public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1);
+    public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0);
     public static final Version CURRENT = V_3_0_0;
diff --git a/libs/core/src/main/java/org/opensearch/core/common/Strings.java b/libs/core/src/main/java/org/opensearch/core/common/Strings.java
index 8fdec670bd9f2..e8379e11ea26a 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/Strings.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/Strings.java
@@ -815,4 +815,17 @@ public static String toLowercaseAscii(String in) {
         }
         return out.toString();
     }
+
+    /**
+     * Check whether every single character in the string is a digit.
+     *
+     * <p>An empty string returns {@code false}.</p>
+     *
+     * @param s the string, must not be null.
+     * @return {@code true} if the string only contains digits, {@code false} otherwise.
+     */
+    public static boolean isDigits(final String s) {
+        return !s.isEmpty() && s.chars().allMatch(Character::isDigit);
+    }
+
 }
diff --git a/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java b/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java
index b79bb6fc89f9e..be7af18b106a8 100644
--- a/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java
+++ b/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java
@@ -114,4 +114,15 @@ public void testToStringToXContentWithOrWithoutParams() {
             containsString("\"color_from_param\":\"blue\"")
         );
     }
+
+    public void testIsDigits() {
+        assertTrue(Strings.isDigits("1"));
+        assertTrue(Strings.isDigits("123"));
+        assertFalse(Strings.isDigits(""));
+        assertFalse(Strings.isDigits("abc"));
+        assertFalse(Strings.isDigits("123a"));
+        assertFalse(Strings.isDigits("0x123"));
+        assertFalse(Strings.isDigits("123.4"));
+        assertFalse(Strings.isDigits("123f"));
+    }
 }
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index c0ff155ce1038..f74de1dc290dd 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -39,9 +39,9 @@ opensearchplugin {
 }
 
 dependencies {
-  api('com.maxmind.geoip2:geoip2:4.2.0')
+  api('com.maxmind.geoip2:geoip2:4.2.1')
   // geoip2 dependencies:
-  api('com.maxmind.db:maxmind-db:3.1.0')
+  api('com.maxmind.db:maxmind-db:3.1.1')
   api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
   api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
 
diff --git a/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1
deleted file mode 100644
index b6bfeeb9da60b..0000000000000
--- a/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-78ff932dc13ac41dd1f0fd9e7405a7f4ad815ce0
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1
new file mode 100644
index 0000000000000..a9dc5483ac727
--- /dev/null
+++ b/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1
@@ -0,0 +1 @@
+9dbf8a8bea88a33e88c46eb3f503721b4bd08b90
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1
deleted file mode 100644
index 9db7c7319af0b..0000000000000
--- a/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2008992ab45d61c7b28a18678b5df82272529da3
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1
new file mode 100644
index 0000000000000..f1a25c6d600c0
--- /dev/null
+++ b/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1
@@ -0,0 +1 @@
+84a2a5d322fef1f1e82f05095683c8d53902baf1
\ No newline at end of file
diff --git a/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java b/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java
index 9446ec1228532..ac6b87aefb3ec 100644
--- a/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java
+++ b/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java
@@ -35,6 +35,7 @@
 import com.maxmind.geoip2.model.AbstractResponse;
 
 import org.opensearch.common.network.InetAddresses;
+import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.env.TestEnvironment;
 import org.opensearch.ingest.Processor;
@@ -126,6 +127,12 @@ public void testAllowListNotSpecified() throws IOException {
         }
     }
 
+    public void testSettingsRegistration() {
+        final IngestGeoIpModulePlugin plugin = new IngestGeoIpModulePlugin();
+        final List<Setting<?>> settings = plugin.getSettings();
+        assertTrue(settings.contains(IngestGeoIpModulePlugin.PROCESSORS_ALLOWLIST_SETTING));
+    }
+
     private void runAllowListTest(List<String> allowList) throws IOException {
         Settings.Builder settingsBuilder = Settings.builder();
         createDb(settingsBuilder);
diff --git a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java
index bac90d20b44e1..f5e2e34278880 100644
--- a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java
+++ b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java
@@ -44,6 +44,7 @@
 import java.nio.file.Path;
 import java.nio.file.PathMatcher;
 import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -152,6 +153,6 @@ static Map<String, UserAgentParser> createUserAgentParsers(Path userAgentConfigD
 
     @Override
     public List<Setting<?>> getSettings() {
-        return Collections.singletonList(CACHE_SIZE_SETTING);
+        return Arrays.asList(CACHE_SIZE_SETTING, PROCESSORS_ALLOWLIST_SETTING);
     }
 }
diff --git a/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java b/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java
index 31fdafff1188a..563158026c37f 100644
--- a/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java
+++ b/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.ingest.useragent;
 
+import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.env.TestEnvironment;
 import org.opensearch.ingest.Processor;
@@ -89,6 +90,12 @@ public void testAllowListNotSpecified() throws IOException {
         }
     }
 
+    public void testSettingsRegistration() {
+        final IngestUserAgentModulePlugin plugin = new IngestUserAgentModulePlugin();
+        final List<Setting<?>> settings = plugin.getSettings();
+        assertTrue(settings.contains(IngestUserAgentModulePlugin.PROCESSORS_ALLOWLIST_SETTING));
+    }
+
     private void runAllowListTest(List<String> allowList) throws IOException {
         final Settings settings = settingsBuilder.putList(IngestUserAgentModulePlugin.PROCESSORS_ALLOWLIST_SETTING.getKey(), allowList)
             .build();
diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java
index 366e848416328..f08815ebbbd1e 100644
--- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java
@@ -264,7 +264,15 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) {
             }
             ft.setPrefixField(prefixFieldType);
             ft.setShingleFields(shingleFieldTypes);
-            return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), prefixFieldMapper, shingleFieldMappers, this);
+            return new SearchAsYouTypeFieldMapper(
+                name,
+                ft,
+                multiFieldsBuilder.build(this, context),
+                copyTo.build(),
+                prefixFieldMapper,
+                shingleFieldMappers,
+                this
+            );
         }
     }
 
@@ -623,12 +631,13 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew
     public SearchAsYouTypeFieldMapper(
         String simpleName,
         SearchAsYouTypeFieldType mappedFieldType,
+        MultiFields multiFields,
         CopyTo copyTo,
         PrefixFieldMapper prefixField,
         ShingleFieldMapper[] shingleFields,
         Builder builder
     ) {
-        super(simpleName, mappedFieldType, MultiFields.empty(), copyTo);
+        super(simpleName, mappedFieldType, multiFields, copyTo);
         this.prefixField = prefixField;
         this.shingleFields = shingleFields;
         this.maxShingleSize = builder.maxShingleSize.getValue();
diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
index f55ad2e9d659c..7746cb714a019 100644
--- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
+++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
@@ -298,6 +298,20 @@ private void assertMultiField(int shingleSize) throws IOException {
         }
     }
 
+    public void testSubField() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(
+                b -> b.field("type", "search_as_you_type")
+                    .startObject("fields")
+                    .startObject("subField")
+                    .field("type", "keyword")
+                    .endObject()
+                    .endObject()
+            )
+        );
+        assertThat(mapperService.fieldType("field.subField"), instanceOf(KeywordFieldMapper.KeywordFieldType.class));
+    }
+
     public void testIndexOptions() throws IOException {
         DocumentMapper mapper = createDocumentMapper(
             fieldMapping(b -> b.field("type", "search_as_you_type").field("index_options", "offsets"))
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1
deleted file mode 100644
index 5c26883046fed..0000000000000
--- a/modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bdc12df04bb6858890b8aa108060b5b365a26102
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..8cb83fc367d78
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+f1d77d15c0b781cd9395a2a956262766fd0c7602
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1
deleted file mode 100644
index 1fd224fdd0b44..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..1be26fee34d46
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+5a49dfa2828d64bf756f670e63259115332744cf
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1
deleted file mode 100644
index 22d35128c3ad5..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-81af1040bfa977f98dd0e1bd9639513ea862ca04
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..e683773245716
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+fbce5a53884275662e68aaad70f88bf7e5d04164
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1
deleted file mode 100644
index d4767d06b22bf..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7fa28b510f0f16f4d5d7188b86bef59e048f62f9
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..2c18924e33c62
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+19ae07fdf99142a70338f8cda70a3d2edbc8e80a
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1
deleted file mode 100644
index 47af3100f0f2d..0000000000000
--- a/modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b2798069092a981a832b7510d0462ee9efb7a80e
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..74ce939dc6190
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+862712e292b162c8ccaa7847a6a54df8178f77e5
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1
deleted file mode 100644
index 8b30272861770..0000000000000
--- a/modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3d5e2d5bcc6baeeb8c13a230980c6132a778e036
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..c431976b6fbd2
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+e56fbde4b9aa628eed15a5dbfbeb97877db88146
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1
deleted file mode 100644
index 1a094fa19a623..0000000000000
--- a/modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-58a631d9d44c4ed7cc0dcc9cffa6641da9374d72
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..e8080a5b2acb1
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+10b23784b23d6a948930f52ba82874f1291b5873
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1
deleted file mode 100644
index 5fbfde0836e0c..0000000000000
--- a/modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-77cd136dd3843f5e7cbcf68c824975d745c49ddb
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..fb2d518789a18
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+e0225a575f487904be8517092cbd74e01913533c
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1
deleted file mode 100644
index 8dad0e3104dc8..0000000000000
--- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b50ff619cdcdc48e748cba3405c9988529f28f60
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..a80b9e51be74b
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+d1171bb99411f282068f49d780cedf8c9adeabfd
\ No newline at end of file
diff --git a/plugins/analysis-phonenumber/build.gradle b/plugins/analysis-phonenumber/build.gradle
new file mode 100644
index 0000000000000..c9913b36f8508
--- /dev/null
+++ b/plugins/analysis-phonenumber/build.gradle
@@ -0,0 +1,21 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+apply plugin: 'opensearch.yaml-rest-test'
+
+opensearchplugin {
+  description 'Adds an analyzer for phone numbers to OpenSearch.'
+  classname 'org.opensearch.analysis.phone.PhoneNumberAnalysisPlugin'
+}
+
+dependencies {
+  implementation group: 'com.googlecode.libphonenumber', name: 'libphonenumber', version: '8.13.45'
+}
diff --git a/plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1 b/plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1
new file mode 100644
index 0000000000000..00d393482ee49
--- /dev/null
+++ b/plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1
@@ -0,0 +1 @@
+bfac00f71616796abc7d8b135dda12558a0ccee2
\ No newline at end of file
diff --git a/plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt b/plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt
new file mode 100644
index 0000000000000..d9a10c0d8e868
--- /dev/null
+++ b/plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt
@@ -0,0 +1,176 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
diff --git a/plugins/analysis-phonenumber/licenses/libphonenumber-NOTICE.txt b/plugins/analysis-phonenumber/licenses/libphonenumber-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java
new file mode 100644
index 0000000000000..eb12b43f70154
--- /dev/null
+++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java
@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analysis.phone;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.opensearch.index.analysis.AnalyzerProvider;
+import org.opensearch.index.analysis.TokenizerFactory;
+import org.opensearch.indices.analysis.AnalysisModule;
+import org.opensearch.plugins.AnalysisPlugin;
+import org.opensearch.plugins.Plugin;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * This plugin provides an analyzer and tokenizer for fields which contain phone numbers, supporting a variety of formats
+ * (with/without international calling code, different country formats, etc.).
+ */
+public class PhoneNumberAnalysisPlugin extends Plugin implements AnalysisPlugin {
+
+    @Override
+    public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
+        Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> analyzers = new TreeMap<>();
+        analyzers.put(
+            "phone",
+            (indexSettings, environment, name, settings) -> new PhoneNumberAnalyzerProvider(indexSettings, "phone", settings, true)
+        );
+        analyzers.put(
+            "phone-search",
+            (indexSettings, environment, name, settings) -> new PhoneNumberAnalyzerProvider(indexSettings, "phone-search", settings, false)
+        );
+        return analyzers;
+    }
+
+    @Override
+    public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
+        Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizers = new TreeMap<>();
+        tokenizers.put(
+            "phone",
+            (indexSettings, environment, name, settings) -> new PhoneNumberTermTokenizerFactory(indexSettings, "phone", settings, true)
+        );
+        tokenizers.put(
+            "phone-search",
+            (indexSettings, environment, name, settings) -> new PhoneNumberTermTokenizerFactory(
+                indexSettings,
+                "phone-search",
+                settings,
+                false
+            )
+        );
+        return tokenizers;
+    }
+
+}
diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java
new file mode 100644
index 0000000000000..cd945e186b2ba
--- /dev/null
+++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java
@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analysis.phone;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.opensearch.common.settings.Settings;
+
+/**
+ * Analyzer for phone numbers, using {@link PhoneNumberTermTokenizer}.
+ *
+ * <p>
+ * You can use the {@code phone} and {@code phone-search} analyzers on your fields to index phone numbers.
+ * Use {@code phone} (which creates ngrams) for the {@code analyzer} and {@code phone-search} (which doesn't create ngrams)
+ * for the {@code search_analyzer}.
+ * </p>
+ *
+ * <p>
+ * You optionally can specify a region with the {@code phone-region} setting for the phone number which will ensure that
+ * phone numbers without the international  dialling prefix (using {@code +}) are also tokenized correctly.
+ * </p>
+ *
+ * <p>
+ * Note that the tokens will not refer to a specific position in the stream as the tokenizer is expected to be used on strings
+ * containing phone numbers and not arbitrary text with interspersed phone numbers.
+ * </p>
+ */
+public class PhoneNumberAnalyzer extends Analyzer {
+    private final boolean addNgrams;
+    private final Settings settings;
+
+    /**
+     * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search.
+     * @param settings the settings for the analyzer.
+     */
+    public PhoneNumberAnalyzer(final Settings settings, final boolean addNgrams) {
+        this.addNgrams = addNgrams;
+        this.settings = settings;
+    }
+
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName) {
+        final var tokenizer = new PhoneNumberTermTokenizer(this.settings, this.addNgrams);
+        return new Analyzer.TokenStreamComponents(tokenizer, tokenizer);
+    }
+}
diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java
new file mode 100644
index 0000000000000..272a019ba0f9c
--- /dev/null
+++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analysis.phone;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.analysis.AbstractIndexAnalyzerProvider;
+
+/**
+ * Provider for {@link PhoneNumberAnalyzer}.
+ */
+public class PhoneNumberAnalyzerProvider extends AbstractIndexAnalyzerProvider<PhoneNumberAnalyzer> {
+
+    private final PhoneNumberAnalyzer analyzer;
+
+    /**
+     * @param indexSettings the settings of the index.
+     * @param name the analyzer name.
+     * @param settings the settings for the analyzer.
+     * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search.
+     */
+    public PhoneNumberAnalyzerProvider(
+        final IndexSettings indexSettings,
+        final String name,
+        final Settings settings,
+        final boolean addNgrams
+    ) {
+        super(indexSettings, name, settings);
+        this.analyzer = new PhoneNumberAnalyzer(settings, addNgrams);
+    }
+
+    @Override
+    public PhoneNumberAnalyzer get() {
+        return this.analyzer;
+    }
+}
diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java
new file mode 100644
index 0000000000000..6b95594204eb4
--- /dev/null
+++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java
@@ -0,0 +1,157 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analysis.phone;
+
+import com.google.i18n.phonenumbers.NumberParseException;
+import com.google.i18n.phonenumbers.PhoneNumberUtil;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.opensearch.common.io.Streams;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.Strings;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This tokenizes a phone number into its individual parts, using {@link PhoneNumberUtil}.
+ *
+ * <p>
+ * You can use the {@code phone} and {@code phone-search} analyzers on your fields to index phone numbers.
+ * Use {@code phone} (which creates ngrams) for the {@code analyzer} and {@code phone-search} (which doesn't create ngrams)
+ * for the {@code search_analyzer}.
+ * </p>
+ *
+ * <p>
+ * You optionally can specify a region with the {@code phone-region} setting for the phone number which will ensure that
+ * phone numbers without the international  dialling prefix (using {@code +}) are also tokenized correctly.
+ * </p>
+ *
+ * <p>
+ * Note that the tokens will not refer to a specific position in the stream as the tokenizer is expected to be used on strings
+ * containing phone numbers and not arbitrary text with interspersed phone numbers.
+ * </p>
+ */
+public final class PhoneNumberTermTokenizer extends Tokenizer {
+    private final boolean addNgrams;
+    private final Settings settings;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private Iterator<String> tokenIterator;
+
+    /**
+     * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search.
+     * @param settings the settings for the analyzer.
+     */
+    public PhoneNumberTermTokenizer(final Settings settings, final boolean addNgrams) {
+        super();
+        this.addNgrams = addNgrams;
+        this.settings = settings;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public void reset() throws IOException {
+        super.reset();
+        tokenIterator = null;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public boolean incrementToken() throws IOException {
+        clearAttributes();
+        if (tokenIterator == null) {
+            tokenIterator = getTokens().iterator();
+        }
+        if (tokenIterator.hasNext()) {
+            termAtt.append(tokenIterator.next());
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Search for a phone number in the input and tokenize it.
+     *
+     * <p>
+     * The tokens include the full phone number with and without country prefix (if it could be identified) and - if
+     * enabled by {@link #addNgrams} - an ngram of the phone number.
+     * </p>
+     *
+     * @return all tokens (unique, unordered).
+     * @throws IOException in case the input cannot be read.
+     */
+    private Set<String> getTokens() throws IOException {
+        final var tokens = new HashSet<String>();
+
+        var input = Streams.copyToString(this.input);
+
+        tokens.add(input);
+
+        // Rip off the "tel:" or "sip:" prefix
+        if (input.indexOf("tel:") == 0 || input.indexOf("sip:") == 0) {
+            tokens.add(input.substring(0, 4));
+            input = input.substring(4);
+        }
+
+        final var startIndex = input.startsWith("+") ? 1 : 0;
+        // Add the complete input but skip a leading +
+        tokens.add(input.substring(startIndex));
+
+        // Drop anything after @. Most likely there's nothing of interest
+        final var posAt = input.indexOf('@');
+        if (posAt != -1) {
+            input = input.substring(0, posAt);
+
+            // Add a token for the raw unmanipulated address. Note this could be a username (sip) instead of telephone
+            // number so take it as is
+            tokens.add(input.substring(startIndex));
+        }
+
+        // Let google's libphone try to parse it
+        final var phoneUtil = PhoneNumberUtil.getInstance();
+        Optional<String> countryCode = Optional.empty();
+        try {
+            // ZZ is the generic "I don't know the country code" region. Google's libphone library will try to infer it.
+            final var region = this.settings.get("phone-region", "ZZ");
+            final var numberProto = phoneUtil.parse(input, region);
+            if (numberProto != null) {
+                // Libphone likes it!
+                countryCode = Optional.of(String.valueOf(numberProto.getCountryCode()));
+                input = String.valueOf(numberProto.getNationalNumber());
+
+                // Add Country code, extension, and the number as tokens
+                tokens.add(countryCode.get());
+                tokens.add(countryCode.get() + input);
+                if (!Strings.isEmpty(numberProto.getExtension())) {
+                    tokens.add(numberProto.getExtension());
+                }
+
+                tokens.add(input);
+            }
+        } catch (final NumberParseException | StringIndexOutOfBoundsException e) {
+            // Libphone didn't like it, no biggie. We'll just ngram the number as it is.
+        }
+
+        // ngram the phone number, e.g. 19198243333 produces 9, 91, 919, etc
+        if (this.addNgrams && Strings.isDigits(input)) {
+            for (int count = 1; count <= input.length(); ++count) {
+                final var token = input.substring(0, count);
+                tokens.add(token);
+                // If there was a country code, add more ngrams such that 19198243333 produces 19, 191, 1919, etc
+                countryCode.ifPresent(s -> tokens.add(s + token));
+            }
+        }
+
+        return tokens;
+    }
+
+}
diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java
new file mode 100644
index 0000000000000..fde44e15c9667
--- /dev/null
+++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java
@@ -0,0 +1,44 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analysis.phone;
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.analysis.AbstractTokenizerFactory;
+
+/**
+ * Factory for {@link PhoneNumberTermTokenizer}.
+ */
+public class PhoneNumberTermTokenizerFactory extends AbstractTokenizerFactory {
+    private final Settings settings;
+    private final boolean addNgrams;
+
+    /**
+     * @param indexSettings the settings of the index.
+     * @param name the tokenizer name.
+     * @param settings the settings for the analyzer.
+     * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search.
+     */
+    public PhoneNumberTermTokenizerFactory(
+        final IndexSettings indexSettings,
+        final String name,
+        final Settings settings,
+        final boolean addNgrams
+    ) {
+        super(indexSettings, settings, name);
+        this.settings = settings;
+        this.addNgrams = addNgrams;
+    }
+
+    @Override
+    public Tokenizer create() {
+        return new PhoneNumberTermTokenizer(this.settings, this.addNgrams);
+    }
+}
diff --git a/plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java b/plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java
new file mode 100644
index 0000000000000..332f6d21f47d6
--- /dev/null
+++ b/plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java
@@ -0,0 +1,253 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analysis.phone;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.opensearch.index.analysis.AnalysisTestsHelper;
+import org.opensearch.test.OpenSearchTokenStreamTestCase;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.hasItemInArray;
+
+public class PhoneNumberAnalyzerTests extends OpenSearchTokenStreamTestCase {
+    private static final String RESOURCE = "/org/opensearch/analysis/phone/phone_analysis.json";
+
+    private static Analyzer phoneAnalyzer;
+    private static Analyzer phoneSearchAnalyzer;
+    private static Analyzer phoneCHAnalyzer;
+    private static Analyzer phoneSearchCHAnalyzer;
+
+    @BeforeClass
+    public static void beforeClass() throws IOException {
+        final var analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(
+            createTempDir(),
+            RESOURCE,
+            new PhoneNumberAnalysisPlugin()
+        );
+        phoneAnalyzer = analysis.indexAnalyzers.get("phone");
+        assertNotNull(phoneAnalyzer);
+        phoneSearchAnalyzer = analysis.indexAnalyzers.get("phone-search");
+        assertNotNull(phoneSearchAnalyzer);
+        phoneCHAnalyzer = analysis.indexAnalyzers.get("phone-ch");
+        assertNotNull(phoneCHAnalyzer);
+        phoneSearchCHAnalyzer = analysis.indexAnalyzers.get("phone-search-ch");
+        assertNotNull(phoneSearchCHAnalyzer);
+    }
+
+    /**
+     * Test for all tokens which are emitted by the "phone" analyzer.
+     */
+    public void testEuropeDetailled() throws IOException {
+        assertTokensAreInAnyOrder(
+            phoneAnalyzer,
+            "tel:+441344840400",
+            Arrays.asList(
+                "tel:+441344840400",
+                "tel:",
+                "441344840400",
+                "44",
+                "1344840400",
+                "1",
+                "441",
+                "13",
+                "4413",
+                "134",
+                "44134",
+                "1344",
+                "441344",
+                "13448",
+                "4413448",
+                "134484",
+                "44134484",
+                "1344840",
+                "441344840",
+                "13448404",
+                "4413448404",
+                "134484040",
+                "44134484040"
+            )
+        );
+    }
+
+    /**
+     * Test for all tokens which are emitted by the "phone" analyzer.
+     */
+    public void testEuropeDetailledSearch() throws IOException {
+        assertTokensAreInAnyOrder(
+            phoneSearchAnalyzer,
+            "tel:+441344840400",
+            Arrays.asList("tel:+441344840400", "tel:", "441344840400", "44", "1344840400")
+        );
+    }
+
+    public void testEurope() throws IOException {
+        assertTokensInclude("tel:+441344840400", Arrays.asList("44", "1344", "1344840400", "441344840400"));
+    }
+
+    public void testGermanCastle() throws IOException {
+        assertTokensInclude("tel:+498362930830", Arrays.asList("49", "498362930830", "8362930830"));
+    }
+
+    public void testBMWofSydney() throws IOException {
+        assertTokensInclude("tel:+61293344555", Arrays.asList("61", "293344555", "61293344555"));
+    }
+
+    public void testCoffeeShopInIreland() throws IOException {
+        assertTokensInclude("tel:+442890319416", Arrays.asList("44", "289", "2890319416", "442890319416"));
+    }
+
+    public void testTelWithCountryCode() throws IOException {
+        assertTokensInclude("tel:+17177158163", Arrays.asList("1", "717", "7177", "17177158163"));
+    }
+
+    public void testTelWithCountryCode2() throws IOException {
+        assertTokensInclude("tel:+12177148350", Arrays.asList("1", "217", "2177", "2177148350", "12177148350"));
+    }
+
+    public void testNewTollFreeNumber() throws IOException {
+        assertTokensInclude("tel:+18337148350", Arrays.asList("1", "833", "8337", "8337148350", "18337148350"));
+    }
+
+    public void testMissingCountryCode() throws IOException {
+        assertTokensInclude("tel:8177148350", Arrays.asList("817", "8177", "81771", "817714", "8177148350"));
+    }
+
+    public void testSipWithNumericUsername() throws IOException {
+        assertTokensInclude("sip:222@autosbcpc", Arrays.asList("222"));
+    }
+
+    public void testTruncatedNumber() throws IOException {
+        assertTokensInclude("tel:5551234", Arrays.asList("5551234"));
+    }
+
+    public void testSipWithAlphabeticUsername() throws IOException {
+        assertTokensInclude("sip:abc@autosbcpc", Arrays.asList("abc"));
+    }
+
+    public void testGarbageInGarbageOut() throws IOException {
+        assertTokensInclude("test", Arrays.asList("test"));
+    }
+
+    public void testSipWithCountryCode() throws IOException {
+        assertTokensInclude("sip:+14177141363@178.97.105.13;isup-oli=0;pstn-params=808481808882", Arrays.asList("417", "4177", "14177"));
+    }
+
+    public void testSipWithTelephoneExtension() throws IOException {
+        assertTokensInclude("sip:+13169410766;ext=2233@178.17.10.117:8060", Arrays.asList("316", "2233", "1316"));
+    }
+
+    public void testSipWithUsername() throws IOException {
+        assertTokensInclude("sip:JeffSIP@178.12.220.18", Arrays.asList("JeffSIP"));
+    }
+
+    public void testPhoneNumberWithoutPrefix() throws IOException {
+        assertTokensInclude("+14177141363", Arrays.asList("14177141363", "417", "4177", "14177"));
+    }
+
+    public void testSipWithoutDomainPart() throws IOException {
+        assertTokensInclude("sip:+122882", Arrays.asList("122882", "122", "228", "1228", "2288", "12288"));
+    }
+
+    public void testTelPrefix() throws IOException {
+        assertTokensInclude("tel:+1228", Arrays.asList("1228", "122", "228"));
+    }
+
+    public void testNumberPrefix() throws IOException {
+        assertTokensInclude("+1228", Arrays.asList("1228", "122", "228"));
+    }
+
+    public void testInternationalPrefixWithZZ() throws IOException {
+        assertTokensInclude(phoneAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010"));
+    }
+
+    public void testInternationalPrefixWithCH() throws IOException {
+        assertTokensInclude(phoneCHAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010"));
+    }
+
+    public void testNationalPrefixWithCH() throws IOException {
+        // + is equivalent to 00 in Switzerland
+        assertTokensInclude(phoneCHAnalyzer, "0041583161010", Arrays.asList("41", "41583161010", "583161010"));
+    }
+
+    public void testLocalNumberWithCH() throws IOException {
+        // when omitting the international prefix swiss numbers must start with '0'
+        assertTokensInclude(phoneCHAnalyzer, "0583161010", Arrays.asList("41", "41583161010", "583161010"));
+    }
+
+    public void testSearchInternationalPrefixWithZZ() throws IOException {
+        assertTokensInclude(phoneSearchAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010"));
+    }
+
+    public void testSearchInternationalPrefixWithCH() throws IOException {
+        assertTokensInclude(phoneSearchCHAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010"));
+    }
+
+    public void testSearchNationalPrefixWithCH() throws IOException {
+        // + is equivalent to 00 in Switzerland
+        assertTokensInclude(phoneSearchCHAnalyzer, "0041583161010", Arrays.asList("41", "41583161010", "583161010"));
+    }
+
+    public void testSearchLocalNumberWithCH() throws IOException {
+        // when omitting the international prefix swiss numbers must start with '0'
+        assertTokensInclude(phoneSearchCHAnalyzer, "0583161010", Arrays.asList("41", "41583161010", "583161010"));
+    }
+
+    /**
+     * Unlike {@link #assertTokenStreamContents(TokenStream, String[])} this only asserts whether the generated tokens
+     * contain the required ones but does not check for order. Use {@link #assertTokensInclude} if completeness is not needed.
+     */
+    private void assertTokensAreInAnyOrder(final Analyzer analyzer, final String input, final List<String> expectedTokens)
+        throws IOException {
+        final var ts = analyzer.tokenStream("test", input);
+        final var allTokens = getAllTokens(ts).toArray();
+        assertThat(allTokens, arrayContainingInAnyOrder(expectedTokens.toArray()));
+    }
+
+    /**
+     * Unlike {@link #assertTokenStreamContents(TokenStream, String[])} this only asserts whether the generated tokens
+     * contain the required ones but does not check for completeness or order.
+     */
+    private void assertTokensInclude(final Analyzer analyzer, final String input, final List<String> expectedTokens) throws IOException {
+        final var ts = analyzer.tokenStream("test", input);
+        final var allTokens = getAllTokens(ts).toArray();
+        for (final var expectedToken : expectedTokens) {
+            assertThat(allTokens, hasItemInArray(expectedToken));
+        }
+    }
+
+    /**
+     * Unlike {@link #assertTokenStreamContents(TokenStream, String[])} this only asserts whether the generated tokens
+     * contain the required ones but does not check for completeness or order.
+     * This uses {@link #phoneAnalyzer}.
+     */
+    private void assertTokensInclude(final String input, final List<String> expectedTokens) throws IOException {
+        this.assertTokensInclude(phoneAnalyzer, input, expectedTokens);
+    }
+
+    private List<String> getAllTokens(final TokenStream ts) throws IOException {
+        final var tokens = new ArrayList<String>();
+        final var termAtt = ts.getAttribute(CharTermAttribute.class);
+        ts.reset();
+        while (ts.incrementToken()) {
+            tokens.add(termAtt.toString());
+        }
+        ts.end();
+        ts.close();
+        return tokens;
+    }
+
+}
diff --git a/plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json b/plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json
new file mode 100644
index 0000000000000..7e45177c57492
--- /dev/null
+++ b/plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json
@@ -0,0 +1,22 @@
+{
+  "index": {
+    "analysis": {
+      "analyzer": {
+        "phone": {
+          "type": "phone"
+        },
+        "phone-search": {
+          "type": "phone-search"
+        },
+        "phone-ch": {
+          "type": "phone",
+          "phone-region": "CH"
+        },
+        "phone-search-ch": {
+          "type": "phone-search",
+          "phone-region": "CH"
+        }
+      }
+    }
+  }
+}
diff --git a/plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java b/plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..d514a3329a1a7
--- /dev/null
+++ b/plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java
@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.analysis.phone;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;
+
+public class PhoneNumberAnalysisClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {
+    public PhoneNumberAnalysisClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return OpenSearchClientYamlSuiteTestCase.createParameters();
+    }
+}
diff --git a/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml
new file mode 100644
index 0000000000000..5bea0cf4650d6
--- /dev/null
+++ b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml
@@ -0,0 +1,8 @@
+"Test that the plugin is loaded in OpenSearch":
+  - do:
+      cat.plugins:
+        local: true
+        h: component
+
+  - match:
+      $body: /^analysis-phonenumber\n$/
diff --git a/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml
new file mode 100644
index 0000000000000..0bd7d2c371bfc
--- /dev/null
+++ b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml
@@ -0,0 +1,56 @@
+# Integration tests for phone analysis components
+#
+---
+"Index phone number content":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              analysis:
+                analyzer:
+                  phone-ch:
+                    type: "phone"
+                    "phone-region": "CH"
+                  phone-search-ch:
+                    type: "phone-search"
+                    "phone-region": "CH"
+          mappings:
+            properties:
+              phone:
+                type: text
+                analyzer: "phone"
+                search_analyzer: "phone-search"
+              phone-ch:
+                type: text
+                analyzer: "phone-ch"
+                search_analyzer: "phone-search-ch"
+
+  - do:
+      index:
+        index:  test
+        id:     1
+        body:   { "phone": "+41 58 316 10 10", "phone-ch": "058 316 10 10" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          query:
+            match:
+              "phone": "+41583161010"
+  - match: { hits.total: 1 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          query:
+            match:
+              "phone-ch": "+41583161010"
+  - match: { hits.total: 1 }
diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java
index af802596ebaa7..2da788242a745 100644
--- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java
+++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java
@@ -13,20 +13,28 @@
 import org.apache.shiro.SecurityUtils;
 import org.apache.shiro.mgt.SecurityManager;
 import org.opensearch.client.Client;
+import org.opensearch.client.node.NodeClient;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.rest.RestStatus;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
 import org.opensearch.identity.PluginSubject;
 import org.opensearch.identity.Subject;
+import org.opensearch.identity.tokens.AuthToken;
 import org.opensearch.identity.tokens.TokenManager;
+import org.opensearch.plugins.ActionPlugin;
 import org.opensearch.plugins.IdentityPlugin;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.rest.BytesRestResponse;
+import org.opensearch.rest.RestChannel;
+import org.opensearch.rest.RestHandler;
+import org.opensearch.rest.RestRequest;
 import org.opensearch.script.ScriptService;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.watcher.ResourceWatcherService;
@@ -34,14 +42,12 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
 
 /**
  * Identity implementation with Shiro
- *
- * @opensearch.experimental
  */
-@ExperimentalApi
-public final class ShiroIdentityPlugin extends Plugin implements IdentityPlugin {
+public final class ShiroIdentityPlugin extends Plugin implements IdentityPlugin, ActionPlugin {
     private Logger log = LogManager.getLogger(this.getClass());
 
     private final Settings settings;
@@ -101,6 +107,37 @@ public TokenManager getTokenManager() {
     }
 
     @Override
+    public UnaryOperator<RestHandler> getRestHandlerWrapper(ThreadContext threadContext) {
+        return AuthcRestHandler::new;
+    }
+
+    class AuthcRestHandler extends RestHandler.Wrapper {
+
+        public AuthcRestHandler(RestHandler original) {
+            super(original);
+        }
+
+        @Override
+        public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+            try {
+                final AuthToken token = ShiroTokenExtractor.extractToken(request);
+                // If no token was found, continue executing the request
+                if (token == null) {
+                    // Authentication did not fail so return true. Authorization is handled at the action level.
+                    super.handleRequest(request, channel, client);
+                    return;
+                }
+                ShiroSubject shiroSubject = (ShiroSubject) getCurrentSubject();
+                shiroSubject.authenticate(token);
+                // Caller was authorized, forward the request to the handler
+                super.handleRequest(request, channel, client);
+            } catch (final Exception e) {
+                final BytesRestResponse bytesRestResponse = new BytesRestResponse(RestStatus.UNAUTHORIZED, e.getMessage());
+                channel.sendResponse(bytesRestResponse);
+            }
+        }
+    }
+
     public PluginSubject getPluginSubject(Plugin plugin) {
         return new ShiroPluginSubject(threadPool);
     }
diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java
index 96cf05ac53a1a..0a809dd6c9071 100644
--- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java
+++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java
@@ -15,8 +15,6 @@
 
 /**
  * OpenSearch specific security manager implementation
- *
- * @opensearch.experimental
  */
 public class ShiroSecurityManager extends DefaultSecurityManager {
 
diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java
index 72a168f23c5cd..73ce3f835fc9b 100644
--- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java
+++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java
@@ -17,8 +17,6 @@
 
 /**
  * Subject backed by Shiro
- *
- * @opensearch.experimental
  */
 public class ShiroSubject implements UserSubject {
     private final ShiroTokenManager authTokenHandler;
diff --git a/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java
similarity index 86%
rename from server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java
rename to plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java
index 4bd3ebdded588..86be5ca664daa 100644
--- a/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java
+++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java
@@ -5,11 +5,13 @@
  * this file be licensed under the Apache-2.0 license or a
  * compatible open source license.
  */
-package org.opensearch.identity.tokens;
+package org.opensearch.identity.shiro;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.core.common.Strings;
+import org.opensearch.identity.tokens.AuthToken;
+import org.opensearch.identity.tokens.BasicAuthToken;
 import org.opensearch.rest.RestRequest;
 
 import java.util.Collections;
@@ -18,9 +20,9 @@
 /**
  * Extracts tokens from RestRequests used for authentication
  */
-public class RestTokenExtractor {
+public class ShiroTokenExtractor {
 
-    private static final Logger logger = LogManager.getLogger(RestTokenExtractor.class);
+    private static final Logger logger = LogManager.getLogger(ShiroTokenExtractor.class);
 
     public final static String AUTH_HEADER_NAME = "Authorization";
 
diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java
index a14215aa7655b..cd54bbf9b3124 100644
--- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java
+++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java
@@ -36,8 +36,6 @@
 
 /**
  * Extracts Shiro's {@link AuthenticationToken} from different types of auth headers
- *
- * @opensearch.experimental
  */
 class ShiroTokenManager implements TokenManager {
 
diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java
index a2cb78425929e..f8113101deb70 100644
--- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java
+++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java
@@ -16,8 +16,6 @@
 
 /**
  * Password matcher for BCrypt
- *
- * @opensearch.experimental
  */
 public class BCryptPasswordMatcher implements CredentialsMatcher {
 
diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java
index ef405a5637ae7..1fc9a1f437a42 100644
--- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java
+++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java
@@ -25,8 +25,6 @@
 
 /**
  * Internal Realm is a custom realm using the internal OpenSearch IdP
- *
- * @opensearch.experimental
  */
 public class OpenSearchRealm extends AuthenticatingRealm {
     private static final String DEFAULT_REALM_NAME = "internal";
@@ -93,7 +91,7 @@ public OpenSearchRealm build() {
     public User getInternalUser(final String principalIdentifier) throws UnknownAccountException {
         final User userRecord = internalUsers.get(principalIdentifier);
         if (userRecord == null) {
-            throw new UnknownAccountException();
+            throw new UnknownAccountException("Incorrect credentials");
         }
         return userRecord;
     }
@@ -131,7 +129,7 @@ protected AuthenticationInfo doGetAuthenticationInfo(final AuthenticationToken t
                 return sai;
             } else {
                 // Bad password
-                throw new IncorrectCredentialsException();
+                throw new IncorrectCredentialsException("Incorrect credentials");
             }
         }
 
diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java
index 35b3348a955d7..1d2d0fed800e2 100644
--- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java
+++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java
@@ -12,8 +12,6 @@
 
 /**
  * A non-volatile and immutable object in the storage.
- *
- * @opensearch.experimental
  */
 public class User {
 
diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java
index bc14410d81de0..a15538e48bd66 100644
--- a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java
+++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java
@@ -13,7 +13,7 @@
 import org.opensearch.identity.IdentityService;
 import org.opensearch.plugins.IdentityPlugin;
 import org.opensearch.test.OpenSearchTestCase;
-import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
 
 import java.util.List;
 
@@ -21,27 +21,27 @@
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThrows;
+import static org.mockito.Mockito.mock;
 
 public class ShiroIdentityPluginTests extends OpenSearchTestCase {
 
     public void testSingleIdentityPluginSucceeds() {
-        TestThreadPool threadPool = new TestThreadPool(getTestName());
         IdentityPlugin identityPlugin1 = new ShiroIdentityPlugin(Settings.EMPTY);
         List<IdentityPlugin> pluginList1 = List.of(identityPlugin1);
-        IdentityService identityService1 = new IdentityService(Settings.EMPTY, threadPool, pluginList1);
+        IdentityService identityService1 = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), pluginList1);
         assertThat(identityService1.getTokenManager(), is(instanceOf(ShiroTokenManager.class)));
-        terminate(threadPool);
     }
 
     public void testMultipleIdentityPluginsFail() {
-        TestThreadPool threadPool = new TestThreadPool(getTestName());
         IdentityPlugin identityPlugin1 = new ShiroIdentityPlugin(Settings.EMPTY);
         IdentityPlugin identityPlugin2 = new ShiroIdentityPlugin(Settings.EMPTY);
         IdentityPlugin identityPlugin3 = new ShiroIdentityPlugin(Settings.EMPTY);
         List<IdentityPlugin> pluginList = List.of(identityPlugin1, identityPlugin2, identityPlugin3);
-        Exception ex = assertThrows(OpenSearchException.class, () -> new IdentityService(Settings.EMPTY, threadPool, pluginList));
+        Exception ex = assertThrows(
+            OpenSearchException.class,
+            () -> new IdentityService(Settings.EMPTY, mock(ThreadPool.class), pluginList)
+        );
         assert (ex.getMessage().contains("Multiple identity plugins are not supported,"));
-        terminate(threadPool);
     }
 
 }
diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java
new file mode 100644
index 0000000000000..4dc398bacb707
--- /dev/null
+++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.identity.shiro;
+
+import org.opensearch.identity.tokens.AuthToken;
+import org.opensearch.identity.tokens.BasicAuthToken;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.test.rest.FakeRestRequest;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class ShiroTokenExtractorTests extends OpenSearchTestCase {
+
+    public void testAuthorizationHeaderExtractionWithBasicAuthToken() {
+        String basicAuthHeader = Base64.getEncoder().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8));
+        RestRequest fakeRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(
+            Map.of(ShiroTokenExtractor.AUTH_HEADER_NAME, List.of(BasicAuthToken.TOKEN_IDENTIFIER + " " + basicAuthHeader))
+        ).build();
+        AuthToken extractedToken = ShiroTokenExtractor.extractToken(fakeRequest);
+        assertThat(extractedToken, instanceOf(BasicAuthToken.class));
+        assertThat(extractedToken.asAuthHeaderValue(), equalTo(basicAuthHeader));
+    }
+
+    public void testAuthorizationHeaderExtractionWithUnknownToken() {
+        String authHeader = "foo";
+        RestRequest fakeRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(
+            Map.of(ShiroTokenExtractor.AUTH_HEADER_NAME, List.of(authHeader))
+        ).build();
+        AuthToken extractedToken = ShiroTokenExtractor.extractToken(fakeRequest);
+        assertNull(extractedToken);
+    }
+}
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 5669c5e559ac8..4baf79e619be9 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -48,7 +48,7 @@ dependencies {
   api 'com.azure:azure-json:1.1.0'
   api 'com.azure:azure-xml:1.1.0'
   api 'com.azure:azure-storage-common:12.25.1'
-  api 'com.azure:azure-core-http-netty:1.15.3'
+  api 'com.azure:azure-core-http-netty:1.15.4'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"
@@ -61,9 +61,9 @@ dependencies {
   // Start of transitive dependencies for azure-identity
   api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0'
   api "net.java.dev.jna:jna-platform:${versions.jna}"
-  api 'com.microsoft.azure:msal4j:1.17.0'
+  api 'com.microsoft.azure:msal4j:1.17.1'
   api 'com.nimbusds:oauth2-oidc-sdk:11.19.1'
-  api 'com.nimbusds:nimbus-jose-jwt:9.40'
+  api 'com.nimbusds:nimbus-jose-jwt:9.41.1'
   api 'com.nimbusds:content-type:2.3'
   api 'com.nimbusds:lang-tag:1.7'
   // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart,
diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1
deleted file mode 100644
index 3cea52ba67ce5..0000000000000
--- a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-03b5bd5f5c16eea71f130119dbfb1fe5239f806a
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1
new file mode 100644
index 0000000000000..97e6fad264294
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1
@@ -0,0 +1 @@
+489a38c9e6efb5ce01fbd276d8cb6c0e89000459
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1
deleted file mode 100644
index 34101c989eecd..0000000000000
--- a/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7d37157da92b719f250b0023234ac9dda922a2a5
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1
new file mode 100644
index 0000000000000..46c14e819b630
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1
@@ -0,0 +1 @@
+4eb31a9919d9b103c548af7e37e6f9d9f6e46dbc
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1
deleted file mode 100644
index a42a41b6387c8..0000000000000
--- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-06724b184ee870ecc4d8fc36931beeb3c387b0ee
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..04a48547adb05
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+35798729ba06670fb4fcd02db98d9577e363992d
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1
deleted file mode 100644
index d4767d06b22bf..0000000000000
--- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7fa28b510f0f16f4d5d7188b86bef59e048f62f9
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..2c18924e33c62
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+19ae07fdf99142a70338f8cda70a3d2edbc8e80a
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1
deleted file mode 100644
index 5291a16c10448..0000000000000
--- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9aed7e78c467d06a47a45b5b27466380a6427e2f
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..acbab117e4c15
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+42b1159cac3d196f6bdbd528e29f0fab9dbaae06
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1
deleted file mode 100644
index cf50574b87da0..0000000000000
--- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b23c87a85451b3b0e7c3e8e89698cea6831a8418
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..891ed0a444dc2
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+a01071edffb4812009312b461ce5f160cdec9b75
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1
deleted file mode 100644
index 24e8177190e04..0000000000000
--- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-375872f1c16bb51aac016ff6ee4f5d28b1288d4d
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..7df7a05cd7345
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+2fed36ff50059ded641fa5064963c4b4313512f3
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1
deleted file mode 100644
index 8dad0e3104dc8..0000000000000
--- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b50ff619cdcdc48e748cba3405c9988529f28f60
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..a80b9e51be74b
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+d1171bb99411f282068f49d780cedf8c9adeabfd
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1
deleted file mode 100644
index 83228caf233cc..0000000000000
--- a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-42b1dfa0360e4062951b070bac52dd8d96fd7b38
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1
new file mode 100644
index 0000000000000..71fa950cb9530
--- /dev/null
+++ b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1
@@ -0,0 +1 @@
+35532a88e1b49a623ec97fd276cc208ea525b6bc
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1
deleted file mode 100644
index 7c36b789e839c..0000000000000
--- a/plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d6b2e543749a86957777a46cf68aaa337cc558cb
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..32022e1b2eaa4
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+08134b298d48eec5ddf2e3674a978b52e4667304
\ No newline at end of file
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index 00decbe4fa9cd..22aa151c92003 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -82,6 +82,8 @@ dependencies {
   api "joda-time:joda-time:${versions.joda}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
 
+  runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
+
   // network stack
   api "io.netty:netty-buffer:${versions.netty}"
   api "io.netty:netty-codec:${versions.netty}"
@@ -111,6 +113,7 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /jackson-.*/, to: 'jackson'
   mapping from: /jaxb-.*/, to: 'jaxb'
   mapping from: /netty-.*/, to: 'netty'
+  mapping from: /log4j-.*/, to: 'log4j'
 }
 
 bundlePlugin {
@@ -510,9 +513,7 @@ thirdPartyAudit {
     'org.jboss.marshalling.MarshallingConfiguration',
     'org.jboss.marshalling.Unmarshaller',
 
-    'org.slf4j.impl.StaticLoggerBinder',
-    'org.slf4j.impl.StaticMDCBinder',
-    'org.slf4j.impl.StaticMarkerBinder',
+    'org.slf4j.ext.EventData',
     'reactor.blockhound.BlockHound$Builder',
     'reactor.blockhound.integration.BlockHoundIntegration',
 
diff --git a/plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1 b/plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..0e22f98daa61c
--- /dev/null
+++ b/plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1
@@ -0,0 +1 @@
+911fdb5b1a1df36719c579ecc6f2957b88bce1ab
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1
deleted file mode 100644
index 5c26883046fed..0000000000000
--- a/plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bdc12df04bb6858890b8aa108060b5b365a26102
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..8cb83fc367d78
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+f1d77d15c0b781cd9395a2a956262766fd0c7602
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1
deleted file mode 100644
index 1fd224fdd0b44..0000000000000
--- a/plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..1be26fee34d46
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+5a49dfa2828d64bf756f670e63259115332744cf
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1
deleted file mode 100644
index 22d35128c3ad5..0000000000000
--- a/plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-81af1040bfa977f98dd0e1bd9639513ea862ca04
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..e683773245716
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+fbce5a53884275662e68aaad70f88bf7e5d04164
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1
deleted file mode 100644
index d4767d06b22bf..0000000000000
--- a/plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7fa28b510f0f16f4d5d7188b86bef59e048f62f9
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..2c18924e33c62
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+19ae07fdf99142a70338f8cda70a3d2edbc8e80a
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1
deleted file mode 100644
index 47af3100f0f2d..0000000000000
--- a/plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b2798069092a981a832b7510d0462ee9efb7a80e
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..74ce939dc6190
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+862712e292b162c8ccaa7847a6a54df8178f77e5
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1
deleted file mode 100644
index 8b30272861770..0000000000000
--- a/plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3d5e2d5bcc6baeeb8c13a230980c6132a778e036
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..c431976b6fbd2
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+e56fbde4b9aa628eed15a5dbfbeb97877db88146
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1
deleted file mode 100644
index 1a094fa19a623..0000000000000
--- a/plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-58a631d9d44c4ed7cc0dcc9cffa6641da9374d72
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..e8080a5b2acb1
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+10b23784b23d6a948930f52ba82874f1291b5873
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1
deleted file mode 100644
index 5fbfde0836e0c..0000000000000
--- a/plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-77cd136dd3843f5e7cbcf68c824975d745c49ddb
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..fb2d518789a18
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+e0225a575f487904be8517092cbd74e01913533c
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1
deleted file mode 100644
index 0196dacfe92ba..0000000000000
--- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-67e590356eb53c20aaabd67f61ae66f628e62e3d
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..7120cd93e5c0d
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+f442c794e6fe89e6974f058bf393353e01fb927d
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1
deleted file mode 100644
index 8dad0e3104dc8..0000000000000
--- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b50ff619cdcdc48e748cba3405c9988529f28f60
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..a80b9e51be74b
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+d1171bb99411f282068f49d780cedf8c9adeabfd
\ No newline at end of file
diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
index c5438d58e679d..944de326d144c 100644
--- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -153,6 +153,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
             // Disable request throttling because some random values in tests might generate too many failures for the S3 client
             .put(S3ClientSettings.USE_THROTTLE_RETRIES_SETTING.getConcreteSettingForNamespace("test").getKey(), false)
             .put(S3ClientSettings.PROXY_TYPE_SETTING.getConcreteSettingForNamespace("test").getKey(), ProxySettings.ProxyType.DIRECT)
+            .put(BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.getKey(), false)
             .put(super.nodeSettings(nodeOrdinal))
             .setSecureSettings(secureSettings);
 
diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java
index 7db9a0d3ba790..f0e40db965646 100644
--- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java
+++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java
@@ -55,6 +55,14 @@
 
 public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
 
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder()
+            .put(super.nodeSettings())
+            .put(BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.getKey(), false)
+            .build();
+    }
+
     @Override
     @Before
     @SuppressForbidden(reason = "Need to set system property here for AWS SDK v2")
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java
index d691cad9c9d03..8bbef168de89c 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java
@@ -18,6 +18,7 @@
 import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration;
 import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
 import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
+import software.amazon.awssdk.core.retry.RetryMode;
 import software.amazon.awssdk.core.retry.RetryPolicy;
 import software.amazon.awssdk.core.retry.backoff.BackoffStrategy;
 import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
@@ -239,7 +240,9 @@ static ClientOverrideConfiguration buildOverrideConfiguration(final S3ClientSett
                 RetryPolicy.builder()
                     .numRetries(clientSettings.maxRetries)
                     .throttlingBackoffStrategy(
-                        clientSettings.throttleRetries ? BackoffStrategy.defaultThrottlingStrategy() : BackoffStrategy.none()
+                        clientSettings.throttleRetries
+                            ? BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD)
+                            : BackoffStrategy.none()
                     )
                     .build()
             )
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
index b489a3cc85037..1a402e8431e25 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
@@ -62,6 +62,7 @@
 import software.amazon.awssdk.services.s3.model.UploadPartRequest;
 import software.amazon.awssdk.services.s3.model.UploadPartResponse;
 import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
+import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher;
 import software.amazon.awssdk.utils.CollectionUtils;
 
 import org.apache.logging.log4j.LogManager;
@@ -90,6 +91,7 @@
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.repositories.s3.async.S3AsyncDeleteHelper;
 import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
 import org.opensearch.repositories.s3.async.UploadRequest;
 import org.opensearch.repositories.s3.utils.HttpRangeUtils;
@@ -109,6 +111,9 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import org.reactivestreams.Subscriber;
+import org.reactivestreams.Subscription;
+
 import static org.opensearch.repositories.s3.S3Repository.MAX_FILE_SIZE;
 import static org.opensearch.repositories.s3.S3Repository.MAX_FILE_SIZE_USING_MULTIPART;
 import static org.opensearch.repositories.s3.S3Repository.MIN_PART_SIZE_USING_MULTIPART;
@@ -875,4 +880,123 @@ CompletableFuture<GetObjectAttributesResponse> getBlobMetadata(S3AsyncClient s3A
 
         return SocketAccess.doPrivileged(() -> s3AsyncClient.getObjectAttributes(getObjectAttributesRequest));
     }
+
+    @Override
+    public void deleteAsync(ActionListener<DeleteResult> completionListener) {
+        try (AmazonAsyncS3Reference asyncClientReference = blobStore.asyncClientReference()) {
+            S3AsyncClient s3AsyncClient = asyncClientReference.get().client();
+
+            ListObjectsV2Request listRequest = ListObjectsV2Request.builder().bucket(blobStore.bucket()).prefix(keyPath).build();
+            ListObjectsV2Publisher listPublisher = s3AsyncClient.listObjectsV2Paginator(listRequest);
+
+            AtomicLong deletedBlobs = new AtomicLong();
+            AtomicLong deletedBytes = new AtomicLong();
+
+            CompletableFuture<Void> listingFuture = new CompletableFuture<>();
+
+            listPublisher.subscribe(new Subscriber<>() {
+                private Subscription subscription;
+                private final List<String> objectsToDelete = new ArrayList<>();
+                private CompletableFuture<Void> deletionChain = CompletableFuture.completedFuture(null);
+
+                @Override
+                public void onSubscribe(Subscription s) {
+                    this.subscription = s;
+                    subscription.request(1);
+                }
+
+                @Override
+                public void onNext(ListObjectsV2Response response) {
+                    response.contents().forEach(s3Object -> {
+                        deletedBlobs.incrementAndGet();
+                        deletedBytes.addAndGet(s3Object.size());
+                        objectsToDelete.add(s3Object.key());
+                    });
+
+                    int bulkDeleteSize = blobStore.getBulkDeletesSize();
+                    if (objectsToDelete.size() >= bulkDeleteSize) {
+                        int fullBatchesCount = objectsToDelete.size() / bulkDeleteSize;
+                        int itemsToDelete = fullBatchesCount * bulkDeleteSize;
+
+                        List<String> batchToDelete = new ArrayList<>(objectsToDelete.subList(0, itemsToDelete));
+                        objectsToDelete.subList(0, itemsToDelete).clear();
+
+                        deletionChain = S3AsyncDeleteHelper.executeDeleteChain(
+                            s3AsyncClient,
+                            blobStore,
+                            batchToDelete,
+                            deletionChain,
+                            () -> subscription.request(1)
+                        );
+                    } else {
+                        subscription.request(1);
+                    }
+                }
+
+                @Override
+                public void onError(Throwable t) {
+                    listingFuture.completeExceptionally(new IOException("Failed to list objects for deletion", t));
+                }
+
+                @Override
+                public void onComplete() {
+                    if (!objectsToDelete.isEmpty()) {
+                        deletionChain = S3AsyncDeleteHelper.executeDeleteChain(
+                            s3AsyncClient,
+                            blobStore,
+                            objectsToDelete,
+                            deletionChain,
+                            null
+                        );
+                    }
+                    deletionChain.whenComplete((v, throwable) -> {
+                        if (throwable != null) {
+                            listingFuture.completeExceptionally(throwable);
+                        } else {
+                            listingFuture.complete(null);
+                        }
+                    });
+                }
+            });
+
+            listingFuture.whenComplete((v, throwable) -> {
+                if (throwable != null) {
+                    completionListener.onFailure(
+                        throwable instanceof Exception
+                            ? (Exception) throwable
+                            : new IOException("Unexpected error during async deletion", throwable)
+                    );
+                } else {
+                    completionListener.onResponse(new DeleteResult(deletedBlobs.get(), deletedBytes.get()));
+                }
+            });
+        } catch (Exception e) {
+            completionListener.onFailure(new IOException("Failed to initiate async deletion", e));
+        }
+    }
+
+    @Override
+    public void deleteBlobsAsyncIgnoringIfNotExists(List<String> blobNames, ActionListener<Void> completionListener) {
+        if (blobNames.isEmpty()) {
+            completionListener.onResponse(null);
+            return;
+        }
+
+        try (AmazonAsyncS3Reference asyncClientReference = blobStore.asyncClientReference()) {
+            S3AsyncClient s3AsyncClient = asyncClientReference.get().client();
+
+            List<String> keysToDelete = blobNames.stream().map(this::buildKey).collect(Collectors.toList());
+
+            S3AsyncDeleteHelper.executeDeleteChain(s3AsyncClient, blobStore, keysToDelete, CompletableFuture.completedFuture(null), null)
+                .whenComplete((v, throwable) -> {
+                    if (throwable != null) {
+                        completionListener.onFailure(new IOException("Failed to delete blobs " + blobNames, throwable));
+                    } else {
+                        completionListener.onResponse(null);
+                    }
+                });
+        } catch (Exception e) {
+            completionListener.onFailure(new IOException("Failed to initiate async blob deletion", e));
+        }
+    }
 }
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
index f688be9216b8f..90bfa11e18481 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
@@ -63,7 +63,7 @@
 import static org.opensearch.repositories.s3.S3Repository.STORAGE_CLASS_SETTING;
 import static org.opensearch.repositories.s3.S3Repository.UPLOAD_RETRY_ENABLED;
 
-class S3BlobStore implements BlobStore {
+public class S3BlobStore implements BlobStore {
 
     private static final Logger logger = LogManager.getLogger(S3BlobStore.class);
 
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java
index fe81da31432f4..3d5e121778ba9 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java
@@ -42,6 +42,7 @@
 import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
 import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
 import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.retry.RetryMode;
 import software.amazon.awssdk.core.retry.RetryPolicy;
 import software.amazon.awssdk.core.retry.backoff.BackoffStrategy;
 import software.amazon.awssdk.http.SystemPropertyTlsKeyManagersProvider;
@@ -330,6 +331,8 @@ static ClientOverrideConfiguration buildOverrideConfiguration(final S3ClientSett
         );
         if (!clientSettings.throttleRetries) {
             retryPolicy.throttlingBackoffStrategy(BackoffStrategy.none());
+        } else {
+            retryPolicy.throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD));
         }
         return clientOverrideConfiguration.retryPolicy(retryPolicy.build()).build();
     }
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java
index 8d2772d42ebca..9f73c67df3b18 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java
@@ -95,6 +95,10 @@ public void publish(MetricCollection metricCollection) {
         public void close() {}
     };
 
+    public MetricPublisher getDeleteObjectsMetricPublisher() {
+        return deleteObjectsMetricPublisher;
+    }
+
     public MetricPublisher getObjectMetricPublisher = new MetricPublisher() {
         @Override
         public void publish(MetricCollection metricCollection) {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java
new file mode 100644
index 0000000000000..eed95c0e68ef1
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java
@@ -0,0 +1,95 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3.async;
+
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.model.Delete;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.repositories.s3.S3BlobStore;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+public class S3AsyncDeleteHelper {
+    private static final Logger logger = LogManager.getLogger(S3AsyncDeleteHelper.class);
+
+    public static CompletableFuture<Void> executeDeleteChain(
+        S3AsyncClient s3AsyncClient,
+        S3BlobStore blobStore,
+        List<String> objectsToDelete,
+        CompletableFuture<Void> currentChain,
+        Runnable afterDeleteAction
+    ) {
+        List<List<String>> batches = createDeleteBatches(objectsToDelete, blobStore.getBulkDeletesSize());
+        CompletableFuture<Void> newChain = currentChain.thenCompose(v -> executeDeleteBatches(s3AsyncClient, blobStore, batches));
+        if (afterDeleteAction != null) {
+            newChain = newChain.thenRun(afterDeleteAction);
+        }
+        return newChain;
+    }
+
+    static List<List<String>> createDeleteBatches(List<String> keys, int bulkDeleteSize) {
+        List<List<String>> batches = new ArrayList<>();
+        for (int i = 0; i < keys.size(); i += bulkDeleteSize) {
+            batches.add(keys.subList(i, Math.min(keys.size(), i + bulkDeleteSize)));
+        }
+        return batches;
+    }
+
+    static CompletableFuture<Void> executeDeleteBatches(S3AsyncClient s3AsyncClient, S3BlobStore blobStore, List<List<String>> batches) {
+        CompletableFuture<Void> allDeletesFuture = CompletableFuture.completedFuture(null);
+
+        for (List<String> batch : batches) {
+            allDeletesFuture = allDeletesFuture.thenCompose(v -> executeSingleDeleteBatch(s3AsyncClient, blobStore, batch));
+        }
+
+        return allDeletesFuture;
+    }
+
+    static CompletableFuture<Void> executeSingleDeleteBatch(S3AsyncClient s3AsyncClient, S3BlobStore blobStore, List<String> batch) {
+        DeleteObjectsRequest deleteRequest = bulkDelete(blobStore.bucket(), batch, blobStore);
+        return s3AsyncClient.deleteObjects(deleteRequest).thenApply(S3AsyncDeleteHelper::processDeleteResponse);
+    }
+
+    static Void processDeleteResponse(DeleteObjectsResponse deleteObjectsResponse) {
+        if (!deleteObjectsResponse.errors().isEmpty()) {
+            logger.warn(
+                () -> new ParameterizedMessage(
+                    "Failed to delete some blobs {}",
+                    deleteObjectsResponse.errors()
+                        .stream()
+                        .map(s3Error -> "[" + s3Error.key() + "][" + s3Error.code() + "][" + s3Error.message() + "]")
+                        .collect(Collectors.toList())
+                )
+            );
+        }
+        return null;
+    }
+
+    static DeleteObjectsRequest bulkDelete(String bucket, List<String> blobs, S3BlobStore blobStore) {
+        return DeleteObjectsRequest.builder()
+            .bucket(bucket)
+            .delete(
+                Delete.builder()
+                    .objects(blobs.stream().map(blob -> ObjectIdentifier.builder().key(blob).build()).collect(Collectors.toList()))
+                    .quiet(true)
+                    .build()
+            )
+            .overrideConfiguration(o -> o.addMetricPublisher(blobStore.getStatsMetricPublisher().getDeleteObjectsMetricPublisher()))
+            .build();
+    }
+}
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java
index b80b857644f2a..e7312157d7a33 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java
@@ -35,6 +35,7 @@
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
 import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.core.retry.RetryMode;
 import software.amazon.awssdk.core.retry.backoff.BackoffStrategy;
 import software.amazon.awssdk.http.apache.ProxyConfiguration;
 
@@ -364,7 +365,7 @@ private void launchAWSConfigurationTest(
         if (expectedUseThrottleRetries) {
             assertThat(
                 clientOverrideConfiguration.retryPolicy().get().throttlingBackoffStrategy(),
-                is(BackoffStrategy.defaultThrottlingStrategy())
+                is(BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD))
             );
         } else {
             assertThat(clientOverrideConfiguration.retryPolicy().get().throttlingBackoffStrategy(), is(BackoffStrategy.none()));
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
index 654d8a72690c4..2cb11541d924f 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
@@ -69,6 +69,7 @@
 import software.amazon.awssdk.services.s3.model.UploadPartRequest;
 import software.amazon.awssdk.services.s3.model.UploadPartResponse;
 import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
+import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher;
 
 import org.opensearch.action.LatchedActionListener;
 import org.opensearch.common.blobstore.BlobContainer;
@@ -99,22 +100,28 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
+import org.reactivestreams.Subscriber;
+import org.reactivestreams.Subscription;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
 
 public class S3BlobStoreContainerTests extends OpenSearchTestCase {
@@ -1275,6 +1282,504 @@ public void testTransformResponseToInputStreamContainer() throws Exception {
         assertEquals(inputStream.available(), inputStreamContainer.getInputStream().available());
     }
 
+    public void testDeleteAsync() throws Exception {
+        for (int i = 0; i < 100; i++) {
+            testDeleteAsync(i + 1);
+        }
+    }
+
+    private void testDeleteAsync(int bulkDeleteSize) throws InterruptedException {
+        final String bucketName = randomAlphaOfLengthBetween(1, 10);
+        final BlobPath blobPath = new BlobPath();
+
+        final S3BlobStore blobStore = mock(S3BlobStore.class);
+        when(blobStore.bucket()).thenReturn(bucketName);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
+        when(blobStore.getBulkDeletesSize()).thenReturn(bulkDeleteSize);
+
+        final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
+        final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class);
+        when(blobStore.asyncClientReference()).thenReturn(asyncClientReference);
+        AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create(
+            s3AsyncClient,
+            s3AsyncClient,
+            s3AsyncClient,
+            null
+        );
+        when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials);
+
+        final List<S3Object> s3Objects = new ArrayList<>();
+        int numObjects = randomIntBetween(20, 100);
+        long totalSize = 0;
+        for (int i = 0; i < numObjects; i++) {
+            long size = randomIntBetween(1, 100);
+            s3Objects.add(S3Object.builder().key(randomAlphaOfLength(10)).size(size).build());
+            totalSize += size;
+        }
+
+        final List<ListObjectsV2Response> responseList = new ArrayList<>();
+        int size = 0;
+        while (size < numObjects) {
+            int toAdd = randomIntBetween(10, 20);
+            int endIndex = Math.min(numObjects, size + toAdd);
+            responseList.add(ListObjectsV2Response.builder().contents(s3Objects.subList(size, endIndex)).build());
+            size = endIndex;
+        }
+        int expectedDeletedObjectsCall = numObjects / bulkDeleteSize + (numObjects % bulkDeleteSize > 0 ? 1 : 0);
+
+        final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class);
+        AtomicInteger counter = new AtomicInteger();
+        doAnswer(invocation -> {
+            Subscriber<? super ListObjectsV2Response> subscriber = invocation.getArgument(0);
+            subscriber.onSubscribe(new Subscription() {
+                @Override
+                public void request(long n) {
+                    int currentCounter = counter.getAndIncrement();
+                    if (currentCounter < responseList.size()) {
+                        subscriber.onNext(responseList.get(currentCounter));
+                    }
+                    if (currentCounter == responseList.size()) {
+                        subscriber.onComplete();
+                    }
+                }
+
+                @Override
+                public void cancel() {}
+            });
+            return null;
+        }).when(listPublisher).subscribe(ArgumentMatchers.<Subscriber<ListObjectsV2Response>>any());
+        when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher);
+
+        when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(
+            CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build())
+        );
+
+        final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<DeleteResult> deleteResultRef = new AtomicReference<>();
+        blobContainer.deleteAsync(new ActionListener<>() {
+            @Override
+            public void onResponse(DeleteResult deleteResult) {
+                deleteResultRef.set(deleteResult);
+                latch.countDown();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.error("exception during deleteAsync", e);
+                fail("Unexpected failure: " + e.getMessage());
+            }
+        });
+
+        latch.await();
+
+        DeleteResult deleteResult = deleteResultRef.get();
+        assertEquals(numObjects, deleteResult.blobsDeleted());
+        assertEquals(totalSize, deleteResult.bytesDeleted());
+
+        verify(s3AsyncClient, times(1)).listObjectsV2Paginator(any(ListObjectsV2Request.class));
+        verify(s3AsyncClient, times(expectedDeletedObjectsCall)).deleteObjects(any(DeleteObjectsRequest.class));
+    }
+
+    public void testDeleteAsyncFailure() throws Exception {
+        final String bucketName = randomAlphaOfLengthBetween(1, 10);
+        final BlobPath blobPath = new BlobPath();
+
+        final S3BlobStore blobStore = mock(S3BlobStore.class);
+        when(blobStore.bucket()).thenReturn(bucketName);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
+        when(blobStore.getBulkDeletesSize()).thenReturn(1000);
+
+        final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
+        final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class);
+        when(blobStore.asyncClientReference()).thenReturn(asyncClientReference);
+        AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create(
+            s3AsyncClient,
+            s3AsyncClient,
+            s3AsyncClient,
+            null
+        );
+        when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials);
+
+        // Simulate a failure in listObjectsV2Paginator
+        RuntimeException simulatedFailure = new RuntimeException("Simulated failure");
+        when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenThrow(simulatedFailure);
+
+        final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+        blobContainer.deleteAsync(new ActionListener<>() {
+            @Override
+            public void onResponse(DeleteResult deleteResult) {
+                fail("Expected a failure, but got a success response");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exceptionRef.set(e);
+                latch.countDown();
+            }
+        });
+
+        latch.await();
+
+        assertNotNull(exceptionRef.get());
+        assertEquals(IOException.class, exceptionRef.get().getClass());
+        assertEquals("Failed to initiate async deletion", exceptionRef.get().getMessage());
+        assertEquals(simulatedFailure, exceptionRef.get().getCause());
+    }
+
+    public void testDeleteAsyncListingError() throws Exception {
+        final String bucketName = randomAlphaOfLengthBetween(1, 10);
+        final BlobPath blobPath = new BlobPath();
+
+        final S3BlobStore blobStore = mock(S3BlobStore.class);
+        when(blobStore.bucket()).thenReturn(bucketName);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
+        when(blobStore.getBulkDeletesSize()).thenReturn(1000);
+
+        final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
+        final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class);
+        when(blobStore.asyncClientReference()).thenReturn(asyncClientReference);
+        AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create(
+            s3AsyncClient,
+            s3AsyncClient,
+            s3AsyncClient,
+            null
+        );
+        when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials);
+
+        final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class);
+        doAnswer(invocation -> {
+            Subscriber<? super ListObjectsV2Response> subscriber = invocation.getArgument(0);
+            subscriber.onSubscribe(new Subscription() {
+                @Override
+                public void request(long n) {
+                    subscriber.onError(new RuntimeException("Simulated listing error"));
+                }
+
+                @Override
+                public void cancel() {}
+            });
+            return null;
+        }).when(listPublisher).subscribe(ArgumentMatchers.<Subscriber<ListObjectsV2Response>>any());
+        when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher);
+
+        final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+        blobContainer.deleteAsync(new ActionListener<>() {
+            @Override
+            public void onResponse(DeleteResult deleteResult) {
+                fail("Expected a failure, but got a success response");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exceptionRef.set(e);
+                latch.countDown();
+            }
+        });
+
+        latch.await();
+
+        assertNotNull(exceptionRef.get());
+        assertEquals(IOException.class, exceptionRef.get().getClass());
+        assertEquals("Failed to list objects for deletion", exceptionRef.get().getMessage());
+        assertNotNull(exceptionRef.get().getCause());
+        assertEquals("Simulated listing error", exceptionRef.get().getCause().getMessage());
+    }
+
+    public void testDeleteAsyncDeletionError() throws Exception {
+        final String bucketName = randomAlphaOfLengthBetween(1, 10);
+        final BlobPath blobPath = new BlobPath();
+
+        final S3BlobStore blobStore = mock(S3BlobStore.class);
+        when(blobStore.bucket()).thenReturn(bucketName);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
+        when(blobStore.getBulkDeletesSize()).thenReturn(1000);
+
+        final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
+        final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class);
+        when(blobStore.asyncClientReference()).thenReturn(asyncClientReference);
+        AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create(
+            s3AsyncClient,
+            s3AsyncClient,
+            s3AsyncClient,
+            null
+        );
+        when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials);
+
+        final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class);
+        doAnswer(invocation -> {
+            Subscriber<? super ListObjectsV2Response> subscriber = invocation.getArgument(0);
+            subscriber.onSubscribe(new Subscription() {
+                @Override
+                public void request(long n) {
+                    subscriber.onNext(
+                        ListObjectsV2Response.builder().contents(S3Object.builder().key("test-key").size(100L).build()).build()
+                    );
+                    subscriber.onComplete();
+                }
+
+                @Override
+                public void cancel() {}
+            });
+            return null;
+        }).when(listPublisher).subscribe(ArgumentMatchers.<Subscriber<ListObjectsV2Response>>any());
+        when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher);
+
+        // Simulate a failure in deleteObjects
+        CompletableFuture<DeleteObjectsResponse> failedFuture = new CompletableFuture<>();
+        failedFuture.completeExceptionally(new RuntimeException("Simulated delete error"));
+        when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(failedFuture);
+
+        final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+        blobContainer.deleteAsync(new ActionListener<>() {
+            @Override
+            public void onResponse(DeleteResult deleteResult) {
+                fail("Expected a failure, but got a success response");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exceptionRef.set(e);
+                latch.countDown();
+            }
+        });
+
+        latch.await();
+
+        assertNotNull(exceptionRef.get());
+        assertEquals(CompletionException.class, exceptionRef.get().getClass());
+        assertEquals("java.lang.RuntimeException: Simulated delete error", exceptionRef.get().getMessage());
+        assertNotNull(exceptionRef.get().getCause());
+        assertEquals("Simulated delete error", exceptionRef.get().getCause().getMessage());
+    }
+
+    public void testDeleteBlobsAsyncIgnoringIfNotExists() throws Exception {
+        final String bucketName = randomAlphaOfLengthBetween(1, 10);
+        final BlobPath blobPath = new BlobPath();
+
+        final S3BlobStore blobStore = mock(S3BlobStore.class);
+        when(blobStore.bucket()).thenReturn(bucketName);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
+        int bulkDeleteSize = randomIntBetween(1, 10);
+        when(blobStore.getBulkDeletesSize()).thenReturn(bulkDeleteSize);
+
+        final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
+        final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class);
+        when(blobStore.asyncClientReference()).thenReturn(asyncClientReference);
+        AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create(
+            s3AsyncClient,
+            s3AsyncClient,
+            s3AsyncClient,
+            null
+        );
+        when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials);
+
+        final List<String> blobNames = new ArrayList<>();
+        int size = randomIntBetween(10, 100);
+        for (int i = 0; i < size; i++) {
+            blobNames.add(randomAlphaOfLength(10));
+        }
+        int expectedDeleteCalls = size / bulkDeleteSize + (size % bulkDeleteSize > 0 ? 1 : 0);
+
+        when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(
+            CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build())
+        );
+
+        final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+        blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener<Void>() {
+            @Override
+            public void onResponse(Void aVoid) {
+                latch.countDown();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exceptionRef.set(e);
+                latch.countDown();
+            }
+        });
+
+        latch.await();
+
+        assertNull(exceptionRef.get());
+
+        ArgumentCaptor<DeleteObjectsRequest> deleteRequestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class);
+        verify(s3AsyncClient, times(expectedDeleteCalls)).deleteObjects(deleteRequestCaptor.capture());
+
+        DeleteObjectsRequest capturedRequest = deleteRequestCaptor.getAllValues().stream().findAny().get();
+        assertEquals(bucketName, capturedRequest.bucket());
+        int totalBlobsDeleted = deleteRequestCaptor.getAllValues()
+            .stream()
+            .map(r -> r.delete().objects().size())
+            .reduce(Integer::sum)
+            .get();
+        assertEquals(blobNames.size(), totalBlobsDeleted);
+        List<String> deletedKeys = deleteRequestCaptor.getAllValues()
+            .stream()
+            .flatMap(r -> r.delete().objects().stream())
+            .map(ObjectIdentifier::key)
+            .collect(Collectors.toList());
+        assertTrue(deletedKeys.containsAll(blobNames));
+    }
+
+    public void testDeleteBlobsAsyncIgnoringIfNotExistsFailure() throws Exception {
+        final String bucketName = randomAlphaOfLengthBetween(1, 10);
+        final BlobPath blobPath = new BlobPath();
+
+        final S3BlobStore blobStore = mock(S3BlobStore.class);
+        when(blobStore.bucket()).thenReturn(bucketName);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
+        when(blobStore.getBulkDeletesSize()).thenReturn(1000);
+
+        final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
+        final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class);
+        when(blobStore.asyncClientReference()).thenReturn(asyncClientReference);
+        AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create(
+            s3AsyncClient,
+            s3AsyncClient,
+            s3AsyncClient,
+            null
+        );
+        when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials);
+
+        // Simulate a failure in deleteObjects
+        CompletableFuture<DeleteObjectsResponse> failedFuture = new CompletableFuture<>();
+        failedFuture.completeExceptionally(new RuntimeException("Simulated delete failure"));
+        when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(failedFuture);
+
+        final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
+
+        List<String> blobNames = Arrays.asList("blob1", "blob2", "blob3");
+
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+        blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener<Void>() {
+            @Override
+            public void onResponse(Void aVoid) {
+                fail("Expected a failure, but got a success response");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exceptionRef.set(e);
+                latch.countDown();
+            }
+        });
+
+        latch.await();
+
+        assertNotNull(exceptionRef.get());
+        assertEquals(IOException.class, exceptionRef.get().getClass());
+        assertEquals("Failed to delete blobs " + blobNames, exceptionRef.get().getMessage());
+        assertNotNull(exceptionRef.get().getCause());
+        assertEquals("java.lang.RuntimeException: Simulated delete failure", exceptionRef.get().getCause().getMessage());
+    }
+
+    public void testDeleteBlobsAsyncIgnoringIfNotExistsWithEmptyList() throws Exception {
+        final String bucketName = randomAlphaOfLengthBetween(1, 10);
+        final BlobPath blobPath = new BlobPath();
+
+        final S3BlobStore blobStore = mock(S3BlobStore.class);
+        when(blobStore.bucket()).thenReturn(bucketName);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
+        when(blobStore.getBulkDeletesSize()).thenReturn(1000);
+
+        final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
+        final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class);
+        when(blobStore.asyncClientReference()).thenReturn(asyncClientReference);
+        AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create(
+            s3AsyncClient,
+            s3AsyncClient,
+            s3AsyncClient,
+            null
+        );
+        when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials);
+
+        final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
+
+        List<String> emptyBlobNames = Collections.emptyList();
+
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicBoolean onResponseCalled = new AtomicBoolean(false);
+        AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+
+        blobContainer.deleteBlobsAsyncIgnoringIfNotExists(emptyBlobNames, new ActionListener<Void>() {
+            @Override
+            public void onResponse(Void aVoid) {
+                onResponseCalled.set(true);
+                latch.countDown();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exceptionRef.set(e);
+                latch.countDown();
+            }
+        });
+
+        latch.await();
+
+        assertTrue("onResponse should have been called", onResponseCalled.get());
+        assertNull("No exception should have been thrown", exceptionRef.get());
+
+        // Verify that no interactions with S3AsyncClient occurred
+        verifyNoInteractions(s3AsyncClient);
+    }
+
+    public void testDeleteBlobsAsyncIgnoringIfNotExistsInitializationFailure() throws Exception {
+        final String bucketName = randomAlphaOfLengthBetween(1, 10);
+        final BlobPath blobPath = new BlobPath();
+
+        final S3BlobStore blobStore = mock(S3BlobStore.class);
+        when(blobStore.bucket()).thenReturn(bucketName);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
+        when(blobStore.getBulkDeletesSize()).thenReturn(1000);
+
+        // Simulate a failure when getting the asyncClientReference
+        RuntimeException simulatedFailure = new RuntimeException("Simulated initialization failure");
+        when(blobStore.asyncClientReference()).thenThrow(simulatedFailure);
+
+        final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
+
+        List<String> blobNames = Arrays.asList("blob1", "blob2", "blob3");
+
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+
+        blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener<Void>() {
+            @Override
+            public void onResponse(Void aVoid) {
+                fail("Expected a failure, but got a success response");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exceptionRef.set(e);
+                latch.countDown();
+            }
+        });
+
+        latch.await();
+
+        assertNotNull("An exception should have been thrown", exceptionRef.get());
+        assertTrue("Exception should be an IOException", exceptionRef.get() instanceof IOException);
+        assertEquals("Failed to initiate async blob deletion", exceptionRef.get().getMessage());
+        assertEquals(simulatedFailure, exceptionRef.get().getCause());
+    }
+
     private void mockObjectResponse(S3AsyncClient s3AsyncClient, String bucketName, String blobName, int objectSize) {
 
         final InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(objectSize));
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java
new file mode 100644
index 0000000000000..d7f924e05cc70
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java
@@ -0,0 +1,236 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3.async;
+
+import software.amazon.awssdk.metrics.MetricPublisher;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.S3Error;
+
+import org.opensearch.repositories.s3.S3BlobStore;
+import org.opensearch.repositories.s3.StatsMetricPublisher;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class S3AsyncDeleteHelperTests extends OpenSearchTestCase {
+
+    @Mock
+    private S3AsyncClient s3AsyncClient;
+
+    @Mock
+    private S3BlobStore blobStore;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        MockitoAnnotations.openMocks(this);
+    }
+
+    public void testExecuteDeleteChain() {
+        List<String> objectsToDelete = Arrays.asList("key1", "key2", "key3");
+        CompletableFuture<Void> currentChain = CompletableFuture.completedFuture(null);
+
+        // Mock the deleteObjects method of S3AsyncClient
+        when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(
+            CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build())
+        );
+
+        // Mock the getBulkDeletesSize method of S3BlobStore
+        when(blobStore.getBulkDeletesSize()).thenReturn(2);
+
+        // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value
+        StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class);
+        MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher);
+        when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher);
+
+        CompletableFuture<Void> newChain = S3AsyncDeleteHelper.executeDeleteChain(
+            s3AsyncClient,
+            blobStore,
+            objectsToDelete,
+            currentChain,
+            null
+        );
+
+        // Verify that the newChain is completed without any exceptions
+        assertNotNull(newChain);
+        assertTrue(newChain.isDone());
+        assertFalse(newChain.isCompletedExceptionally());
+
+        // Verify that the deleteObjects method of S3AsyncClient was called with the expected request
+        ArgumentCaptor<DeleteObjectsRequest> requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class);
+        verify(s3AsyncClient, times(2)).deleteObjects(requestCaptor.capture());
+
+        List<DeleteObjectsRequest> capturedRequests = requestCaptor.getAllValues();
+        assertEquals(2, capturedRequests.size());
+
+        // Verify that the requests have the expected metric publisher added
+        for (DeleteObjectsRequest request : capturedRequests) {
+            assertNotNull(request.overrideConfiguration());
+            assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher));
+        }
+    }
+
+    public void testCreateDeleteBatches() {
+        List<String> keys = Arrays.asList("key1", "key2", "key3", "key4", "key5", "key6");
+        int bulkDeleteSize = 3;
+
+        List<List<String>> batches = S3AsyncDeleteHelper.createDeleteBatches(keys, bulkDeleteSize);
+
+        assertEquals(2, batches.size());
+        assertEquals(Arrays.asList("key1", "key2", "key3"), batches.get(0));
+        assertEquals(Arrays.asList("key4", "key5", "key6"), batches.get(1));
+    }
+
+    public void testExecuteSingleDeleteBatch() throws Exception {
+        List<String> batch = Arrays.asList("key1", "key2");
+        DeleteObjectsResponse expectedResponse = DeleteObjectsResponse.builder().build();
+
+        when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(CompletableFuture.completedFuture(expectedResponse));
+
+        // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value
+        StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class);
+        MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher);
+        when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher);
+
+        CompletableFuture<Void> future = S3AsyncDeleteHelper.executeSingleDeleteBatch(s3AsyncClient, blobStore, batch);
+
+        assertNotNull(future);
+        assertTrue(future.isDone());
+        assertFalse(future.isCompletedExceptionally());
+        future.join(); // Wait for the CompletableFuture to complete
+
+        // Verify that the deleteObjects method of S3AsyncClient was called with the expected request
+        ArgumentCaptor<DeleteObjectsRequest> requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class);
+        verify(s3AsyncClient).deleteObjects(requestCaptor.capture());
+
+        DeleteObjectsRequest capturedRequest = requestCaptor.getValue();
+        assertEquals(blobStore.bucket(), capturedRequest.bucket());
+        assertEquals(batch.size(), capturedRequest.delete().objects().size());
+        assertTrue(capturedRequest.delete().objects().stream().map(ObjectIdentifier::key).collect(Collectors.toList()).containsAll(batch));
+    }
+
+    public void testProcessDeleteResponse() {
+        DeleteObjectsResponse response = DeleteObjectsResponse.builder()
+            .errors(
+                Arrays.asList(
+                    S3Error.builder().key("key1").code("Code1").message("Message1").build(),
+                    S3Error.builder().key("key2").code("Code2").message("Message2").build()
+                )
+            )
+            .build();
+
+        // Call the processDeleteResponse method
+        S3AsyncDeleteHelper.processDeleteResponse(response);
+    }
+
+    public void testBulkDelete() {
+        List<String> blobs = Arrays.asList("key1", "key2", "key3");
+        String bucketName = "my-bucket";
+
+        // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value
+        StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class);
+        MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher);
+        when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher);
+
+        DeleteObjectsRequest request = S3AsyncDeleteHelper.bulkDelete(bucketName, blobs, blobStore);
+
+        assertEquals(bucketName, request.bucket());
+        assertEquals(blobs.size(), request.delete().objects().size());
+        assertTrue(request.delete().objects().stream().map(ObjectIdentifier::key).collect(Collectors.toList()).containsAll(blobs));
+        assertNotNull(request.overrideConfiguration());
+        assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher));
+    }
+
+    public void testExecuteDeleteBatches() {
+        List<List<String>> batches = Arrays.asList(Arrays.asList("key1", "key2"), Arrays.asList("key3", "key4"));
+        DeleteObjectsResponse expectedResponse = DeleteObjectsResponse.builder().build();
+
+        when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(CompletableFuture.completedFuture(expectedResponse));
+
+        // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value
+        StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class);
+        MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher);
+        when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher);
+
+        CompletableFuture<Void> future = S3AsyncDeleteHelper.executeDeleteBatches(s3AsyncClient, blobStore, batches);
+
+        assertNotNull(future);
+        assertTrue(future.isDone());
+        assertFalse(future.isCompletedExceptionally());
+        future.join(); // Wait for the CompletableFuture to complete
+
+        // Verify that the deleteObjects method of S3AsyncClient was called with the expected requests
+        ArgumentCaptor<DeleteObjectsRequest> requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class);
+        verify(s3AsyncClient, times(2)).deleteObjects(requestCaptor.capture());
+
+        List<DeleteObjectsRequest> capturedRequests = requestCaptor.getAllValues();
+        assertEquals(2, capturedRequests.size());
+        for (DeleteObjectsRequest request : capturedRequests) {
+            assertNotNull(request.overrideConfiguration());
+            assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher));
+        }
+    }
+
+    public void testExecuteDeleteChainWithAfterDeleteAction() {
+        List<String> objectsToDelete = Arrays.asList("key1", "key2", "key3");
+        CompletableFuture<Void> currentChain = CompletableFuture.completedFuture(null);
+        Runnable afterDeleteAction = mock(Runnable.class);
+
+        // Mock the deleteObjects method of S3AsyncClient
+        when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(
+            CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build())
+        );
+
+        // Mock the getBulkDeletesSize method of S3BlobStore
+        when(blobStore.getBulkDeletesSize()).thenReturn(2);
+
+        // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value
+        StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class);
+        MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class);
+        when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher);
+        when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher);
+
+        CompletableFuture<Void> newChain = S3AsyncDeleteHelper.executeDeleteChain(
+            s3AsyncClient,
+            blobStore,
+            objectsToDelete,
+            currentChain,
+            afterDeleteAction
+        );
+
+        // Verify that the newChain is completed without any exceptions
+        assertNotNull(newChain);
+        assertTrue(newChain.isDone());
+        assertFalse(newChain.isCompletedExceptionally());
+
+        // Verify that the afterDeleteAction was called
+        verify(afterDeleteAction).run();
+    }
+
+}
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1
deleted file mode 100644
index 5c26883046fed..0000000000000
--- a/plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bdc12df04bb6858890b8aa108060b5b365a26102
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..8cb83fc367d78
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+f1d77d15c0b781cd9395a2a956262766fd0c7602
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1
deleted file mode 100644
index 1fd224fdd0b44..0000000000000
--- a/plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..1be26fee34d46
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+5a49dfa2828d64bf756f670e63259115332744cf
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1
deleted file mode 100644
index 22d35128c3ad5..0000000000000
--- a/plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-81af1040bfa977f98dd0e1bd9639513ea862ca04
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..e683773245716
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+fbce5a53884275662e68aaad70f88bf7e5d04164
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1
deleted file mode 100644
index 47af3100f0f2d..0000000000000
--- a/plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b2798069092a981a832b7510d0462ee9efb7a80e
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..74ce939dc6190
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+862712e292b162c8ccaa7847a6a54df8178f77e5
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1
deleted file mode 100644
index 8b30272861770..0000000000000
--- a/plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3d5e2d5bcc6baeeb8c13a230980c6132a778e036
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..c431976b6fbd2
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+e56fbde4b9aa628eed15a5dbfbeb97877db88146
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1
deleted file mode 100644
index 1a094fa19a623..0000000000000
--- a/plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-58a631d9d44c4ed7cc0dcc9cffa6641da9374d72
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..e8080a5b2acb1
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+10b23784b23d6a948930f52ba82874f1291b5873
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1
deleted file mode 100644
index 5fbfde0836e0c..0000000000000
--- a/plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-77cd136dd3843f5e7cbcf68c824975d745c49ddb
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..fb2d518789a18
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+e0225a575f487904be8517092cbd74e01913533c
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1
deleted file mode 100644
index 5c26883046fed..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bdc12df04bb6858890b8aa108060b5b365a26102
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..8cb83fc367d78
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+f1d77d15c0b781cd9395a2a956262766fd0c7602
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1
deleted file mode 100644
index 1fd224fdd0b44..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..1be26fee34d46
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+5a49dfa2828d64bf756f670e63259115332744cf
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1
deleted file mode 100644
index a42a41b6387c8..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-06724b184ee870ecc4d8fc36931beeb3c387b0ee
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..04a48547adb05
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+35798729ba06670fb4fcd02db98d9577e363992d
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1
deleted file mode 100644
index 22d35128c3ad5..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-81af1040bfa977f98dd0e1bd9639513ea862ca04
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..e683773245716
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+fbce5a53884275662e68aaad70f88bf7e5d04164
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1
deleted file mode 100644
index d4767d06b22bf..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7fa28b510f0f16f4d5d7188b86bef59e048f62f9
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..2c18924e33c62
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+19ae07fdf99142a70338f8cda70a3d2edbc8e80a
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1
deleted file mode 100644
index 47af3100f0f2d..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b2798069092a981a832b7510d0462ee9efb7a80e
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..74ce939dc6190
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+862712e292b162c8ccaa7847a6a54df8178f77e5
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1
deleted file mode 100644
index 8b30272861770..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3d5e2d5bcc6baeeb8c13a230980c6132a778e036
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..c431976b6fbd2
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+e56fbde4b9aa628eed15a5dbfbeb97877db88146
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1
deleted file mode 100644
index 1a094fa19a623..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-58a631d9d44c4ed7cc0dcc9cffa6641da9374d72
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..e8080a5b2acb1
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+10b23784b23d6a948930f52ba82874f1291b5873
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1
deleted file mode 100644
index 24e8177190e04..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-375872f1c16bb51aac016ff6ee4f5d28b1288d4d
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..7df7a05cd7345
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+2fed36ff50059ded641fa5064963c4b4313512f3
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1
deleted file mode 100644
index 5fbfde0836e0c..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-77cd136dd3843f5e7cbcf68c824975d745c49ddb
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..fb2d518789a18
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+e0225a575f487904be8517092cbd74e01913533c
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1
deleted file mode 100644
index 8dad0e3104dc8..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b50ff619cdcdc48e748cba3405c9988529f28f60
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1
new file mode 100644
index 0000000000000..a80b9e51be74b
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1
@@ -0,0 +1 @@
+d1171bb99411f282068f49d780cedf8c9adeabfd
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java
index 6f3895fffa437..1b60023da0329 100644
--- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java
+++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java
@@ -21,6 +21,7 @@
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
+import java.util.Locale;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
@@ -297,4 +298,31 @@ public void testStreamingBadStream() throws IOException {
         assertThat(streamingResponse.getStatusLine().getStatusCode(), equalTo(200));
         assertThat(streamingResponse.getWarnings(), empty());
     }
+
+    public void testStreamingLargeDocument() throws IOException {
+        final Stream<String> stream = Stream.of(
+            String.format(
+                Locale.getDefault(),
+                "{ \"index\": { \"_index\": \"test-streaming\", \"_id\": \"1\" } }\n{ \"name\": \"%s\"  }\n",
+                randomAlphaOfLength(5000)
+            )
+        );
+
+        final Duration delay = Duration.ofMillis(1);
+        final StreamingRequest<ByteBuffer> streamingRequest = new StreamingRequest<>(
+            "POST",
+            "/_bulk/stream",
+            Flux.fromStream(stream).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8)))
+        );
+
+        final StreamingResponse<ByteBuffer> streamingResponse = client().streamRequest(streamingRequest);
+
+        StepVerifier.create(Flux.from(streamingResponse.getBody()).map(b -> new String(b.array(), StandardCharsets.UTF_8)))
+            .expectNextMatches(s -> s.contains("\"type\":\"illegal_argument_exception\""))
+            .expectComplete()
+            .verify();
+
+        assertThat(streamingResponse.getStatusLine().getStatusCode(), equalTo(200));
+        assertThat(streamingResponse.getWarnings(), empty());
+    }
 }
diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java
index a978af1b11db4..9da456f618ffc 100644
--- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java
+++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java
@@ -16,24 +16,20 @@
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 import org.junit.After;
 
+import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
 
 import reactor.core.publisher.Flux;
-import reactor.test.subscriber.TestSubscriber;
+import reactor.test.StepVerifier;
+import reactor.test.scheduler.VirtualTimeScheduler;
 
-import static org.hamcrest.CoreMatchers.anyOf;
 import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.collection.IsEmptyCollection.empty;
 
 public class ReactorNetty4StreamingStressIT extends OpenSearchRestTestCase {
     @After
@@ -49,6 +45,8 @@ public void tearDown() throws Exception {
     }
 
     public void testCloseClientStreamingRequest() throws Exception {
+        final VirtualTimeScheduler scheduler = VirtualTimeScheduler.create(true);
+
         final AtomicInteger id = new AtomicInteger(0);
         final Stream<String> stream = Stream.generate(
             () -> "{ \"index\": { \"_index\": \"test-stress-streaming\", \"_id\": \""
@@ -57,39 +55,28 @@ public void testCloseClientStreamingRequest() throws Exception {
                 + "{ \"name\": \"josh\"  }\n"
         );
 
+        final Duration delay = Duration.ofMillis(1);
         final StreamingRequest<ByteBuffer> streamingRequest = new StreamingRequest<>(
             "POST",
             "/_bulk/stream",
-            Flux.fromStream(stream).delayElements(Duration.ofMillis(500)).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8)))
+            Flux.fromStream(stream).delayElements(delay, scheduler).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8)))
         );
         streamingRequest.addParameter("refresh", "true");
 
         final StreamingResponse<ByteBuffer> streamingResponse = client().streamRequest(streamingRequest);
-        TestSubscriber<ByteBuffer> subscriber = TestSubscriber.create();
-        streamingResponse.getBody().subscribe(subscriber);
-
-        final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
-        try {
-            // Await for subscriber to receive at least one chunk
-            assertBusy(() -> assertThat(subscriber.getReceivedOnNext(), not(empty())));
-
-            // Close client forceably
-            executor.schedule(() -> {
-                client().close();
-                return null;
-            }, 2, TimeUnit.SECONDS);
+        scheduler.advanceTimeBy(delay); /* emit first element */
 
-            // Await for subscriber to terminate
-            subscriber.block(Duration.ofSeconds(10));
-            assertThat(
-                subscriber.expectTerminalError(),
-                anyOf(instanceOf(InterruptedIOException.class), instanceOf(ConnectionClosedException.class))
-            );
-        } finally {
-            executor.shutdown();
-            if (executor.awaitTermination(1, TimeUnit.SECONDS) == false) {
-                executor.shutdownNow();
-            }
-        }
+        StepVerifier.create(Flux.from(streamingResponse.getBody()).map(b -> new String(b.array(), StandardCharsets.UTF_8)))
+            .expectNextMatches(s -> s.contains("\"result\":\"created\"") && s.contains("\"_id\":\"1\""))
+            .then(() -> {
+                try {
+                    client().close();
+                } catch (final IOException ex) {
+                    throw new UncheckedIOException(ex);
+                }
+            })
+            .then(() -> scheduler.advanceTimeBy(delay))
+            .expectErrorMatches(t -> t instanceof InterruptedIOException || t instanceof ConnectionClosedException)
+            .verify();
     }
 }
diff --git a/qa/remote-clusters/docker-compose.yml b/qa/remote-clusters/docker-compose.yml
index cf6aefcf5c1a3..2112da17efe6e 100644
--- a/qa/remote-clusters/docker-compose.yml
+++ b/qa/remote-clusters/docker-compose.yml
@@ -72,3 +72,11 @@ services:
       - "9600"
     volumes:
       - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+    depends_on:
+      - opensearch-2
+    healthcheck:
+      test: ["CMD", "timeout", "1", "bash", "-c", "cat < /dev/null > /dev/tcp/localhost/9600"]
+      interval: 2s
+      timeout: 1s
+      retries: 5
+      start_period: 15s
diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java
index 1a806b033eb8a..14346b8910c76 100644
--- a/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java
+++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java
@@ -38,7 +38,6 @@ public class IdentityAuthenticationIT extends HttpSmokeTestCase {
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
             .put(super.nodeSettings(nodeOrdinal))
-            .put(FeatureFlags.IDENTITY, "true")
             .build();
     }
 
diff --git a/release-notes/opensearch.release-notes-2.17.1.md b/release-notes/opensearch.release-notes-2.17.1.md
new file mode 100644
index 0000000000000..2ff2bd44c3b1c
--- /dev/null
+++ b/release-notes/opensearch.release-notes-2.17.1.md
@@ -0,0 +1,16 @@
+## 2024-10-01 Version 2.17.1 Release Notes
+
+## [2.17.1]
+### Added
+- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664))
+- Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492))
+
+### Dependencies
+
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml
index 107d298b597d3..825bac9f91649 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml
@@ -69,32 +69,3 @@
         include_defaults: true
 
  - match: {defaults.node.attr.testattr: "test"}
-
----
-"Test set search backpressure mode":
-
-  - do:
-      cluster.put_settings:
-        body:
-          persistent:
-            search_backpressure.mode: "monitor_only"
-
-  - match: {persistent: {search_backpressure: {mode: "monitor_only"}}}
-
----
-"Test set invalid search backpressure mode":
-
-  - skip:
-      version: "- 2.8.99"
-      reason: "Fixed in 2.9.0"
-
-  - do:
-      catch: bad_request
-      cluster.put_settings:
-        body:
-          persistent:
-            search_backpressure.mode: "monitor-only"
-
-  - match: {error.root_cause.0.type: "illegal_argument_exception"}
-  - match: { error.root_cause.0.reason: "Invalid SearchBackpressureMode: monitor-only" }
-  - match: { status: 400 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml
index 83d3d273ebd93..e8da81d7bee41 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml
@@ -62,7 +62,6 @@ setup:
           },
           "required_matches": 1
         }
-
   # Do index refresh
   - do:
       indices.refresh:
@@ -74,7 +73,52 @@ teardown:
   - do:
       indices.delete:
         index: test
-
+---
+"Invalid docs":
+  - skip:
+      version: "- 2.17.99"
+      reason: "parsing of these objects would infinite loop prior to 2.18"
+  # The following documents are invalid.
+  - do:
+      catch: /parsing_exception/
+      index:
+        index: test
+        id: 3
+        body: {
+          "ISBN13": "V12154942123242",
+          "catalog": [ "Arrays in Action" ],
+          "required_matches": 1
+        }
+  - do:
+      catch: /parsing_exception/
+      index:
+        index: test
+        id: 3
+        body: {
+          "ISBN13": "V12154942123242",
+          "catalog": "Strings in Action",
+          "required_matches": 1
+        }
+  - do:
+      catch: /parsing_exception/
+      index:
+        index: test
+        id: 3
+        body: {
+          "ISBN13": "V12154942123242",
+          "catalog": 12345,
+          "required_matches": 1
+        }
+  - do:
+      catch: /parsing_exception/
+      index:
+        index: test
+        id: 3
+        body: {
+          "ISBN13": "V12154942123242",
+          "catalog": [ 12345 ],
+          "required_matches": 1
+        }
 ---
 # Verify that mappings under the catalog field did not expand
 # and no dynamic fields were created.
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml
index 8c8a98b2db22c..0cabbf6f25133 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml
@@ -109,6 +109,19 @@ setup:
 
 ---
 "Date histogram aggregation w/ shared field range test":
+  - do:
+      indices.create:
+        index: dhisto-agg-w-query
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+            refresh_interval: -1
+          mappings:
+            properties:
+              date:
+                type: date
+
   - do:
       bulk:
         refresh: true
@@ -127,6 +140,11 @@ setup:
           - '{"index": {}}'
           - '{"date": "2025-02-14"}'
 
+  - do:
+      indices.forcemerge:
+        index: dhisto-agg-w-query
+        max_num_segments: 1
+
   - do:
       search:
         index: dhisto-agg-w-query
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml
index 1e1d2b0706d6b..93ca668f93a6f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml
@@ -535,8 +535,8 @@ setup:
 ---
 "Double range profiler shows filter rewrite info":
   - skip:
-      version: " - 2.99.99"
-      reason: debug info for filter rewrite added in 3.0.0 (to be backported to 2.15.0)
+      version: " - 2.15.99"
+      reason: debug info for filter rewrite was added in 2.16.0
 
   - do:
       indices.create:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml
new file mode 100644
index 0000000000000..c115dd4751f8f
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml
@@ -0,0 +1,111 @@
+---
+# The test setup includes:
+# - Create nested mapping for test_nested_agg_index index
+# - Index two example documents
+# - nested agg
+
+setup:
+  - do:
+      indices.create:
+        index: test_nested_agg_index
+        body:
+          mappings:
+            properties:
+              a:
+                type: nested
+                properties:
+                  b1:
+                    type: keyword
+                  b2:
+                    type: nested
+                    properties:
+                      c:
+                        type: nested
+                        properties:
+                          d:
+                            type: keyword
+
+  - do:
+      bulk:
+        refresh: true
+        body: |
+          {"index": {"_index": "test_nested_agg_index", "_id": "0"}}
+          {"a": { "b1": "b11", "b2": { "c": { "d": "d1" } }}}
+          {"index": {"_index": "test_nested_agg_index", "_id": "1"}}
+          {"a": { "b1": "b12", "b2": { "c": { "d": "d2" } }}}
+
+---
+# Delete Index when connection is teardown
+teardown:
+  - do:
+      indices.delete:
+        index: test_nested_agg_index
+
+---
+"Supported queries":
+  - skip:
+      version: " - 2.17.99"
+      reason: "fixed in 2.18.0"
+
+  # Verify Document Count
+  - do:
+      search:
+        body: {
+          query: {
+            match_all: { }
+          }
+        }
+
+  - length: { hits.hits: 2 }
+
+  # Verify nested aggregation
+  - do:
+      search:
+        body: {
+          aggs: {
+            nested_agg: {
+              nested: {
+                path: "a"
+              },
+              aggs: {
+                a_b1: {
+                  terms: {
+                    field: "a.b1"
+                  },
+                  aggs: {
+                    "c": {
+                      nested: {
+                        path: "a.b2.c"
+                      },
+                      aggs: {
+                        "d": {
+                          terms: {
+                            field: "a.b2.c.d"
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+
+  - length: { hits.hits: 2 }
+  - match: { aggregations.nested_agg.doc_count: 2 }
+  - length: { aggregations.nested_agg.a_b1.buckets: 2 }
+
+  - match: { aggregations.nested_agg.a_b1.buckets.0.key: "b11" }
+  - match: { aggregations.nested_agg.a_b1.buckets.0.doc_count: 1 }
+  - match: { aggregations.nested_agg.a_b1.buckets.0.c.doc_count: 1 }
+  - length: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets: "1" }
+  - match: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets.0.key: "d1" }
+  - match: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets.0.doc_count: 1 }
+
+  - match: { aggregations.nested_agg.a_b1.buckets.1.key: "b12" }
+  - match: { aggregations.nested_agg.a_b1.buckets.1.doc_count: 1 }
+  - match: { aggregations.nested_agg.a_b1.buckets.1.c.doc_count: 1 }
+  - length: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets: "1" }
+  - match: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets.0.key: "d2" }
+  - match: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets.0.doc_count: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml
new file mode 100644
index 0000000000000..2862dcce072c4
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml
@@ -0,0 +1,340 @@
+---
+"Test set search backpressure mode":
+  - skip:
+      version: "- 2.3.99"
+      reason: "Introduced in 2.4.0"
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            search_backpressure.mode: "monitor_only"
+
+  - match: {persistent: {search_backpressure: {mode: "monitor_only"}}}
+
+---
+"Test set invalid search backpressure mode":
+  - skip:
+      version: "- 2.7.99"
+      reason: "Fixed in 2.8.0"
+
+  - do:
+      catch: bad_request
+      cluster.put_settings:
+        body:
+          persistent:
+            search_backpressure.mode: "monitor-only"
+
+  - match: {error.root_cause.0.type: "illegal_argument_exception"}
+  - match: { error.root_cause.0.reason: "Invalid SearchBackpressureMode: monitor-only" }
+  - match: { status: 400 }
+
+---
+"Test setting search backpressure cancellation settings":
+  - skip:
+      version: "- 2.17.99"
+      reason: "Fixed in 2.18.0"
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.cancellation_burst: 11
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_task.cancellation_burst: "11"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.cancellation_rate: 0.1
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_task.cancellation_rate: "0.1"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.cancellation_ratio: 0.2
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_task.cancellation_ratio: "0.2"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.cancellation_burst: 12
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_shard_task.cancellation_burst: "12"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.cancellation_rate: 0.3
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_shard_task.cancellation_rate: "0.3"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.cancellation_ratio: 0.4
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_shard_task.cancellation_ratio: "0.4"}
+
+---
+"Test setting invalid search backpressure cancellation_rate and cancellation_ratio":
+  - skip:
+      version: "- 2.17.99"
+      reason: "Fixed in 2.18.0"
+
+  - do:
+      catch: /search_backpressure.search_task.cancellation_rate must be > 0/
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.cancellation_rate: 0.0
+
+  - do:
+      catch: /search_backpressure.search_task.cancellation_ratio must be > 0/
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.cancellation_ratio: 0.0
+
+  - do:
+      catch: /search_backpressure.search_shard_task.cancellation_rate must be > 0/
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.cancellation_rate: 0.0
+
+  - do:
+      catch: /search_backpressure.search_shard_task.cancellation_ratio must be > 0/
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.cancellation_ratio: 0.0
+
+---
+"Test setting search backpressure node settings":
+  - skip:
+      version: "- 2.3.99"
+      reason: "Introduced in 2.4.0"
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.node_duress.num_successive_breaches: 10
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.node_duress.num_successive_breaches: "10"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.node_duress.cpu_threshold: 0.85
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.node_duress.cpu_threshold: "0.85"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.node_duress.heap_threshold: 0.75
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.node_duress.heap_threshold: "0.75"}
+
+
+---
+"Test setting search backpressure search_task settings":
+  - skip:
+      version: "- 2.3.99"
+      reason: "Introduced in 2.4.0"
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.elapsed_time_millis_threshold: 50000
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_task.elapsed_time_millis_threshold: "50000"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.heap_percent_threshold: 0.01
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_task.heap_percent_threshold: "0.01"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.total_heap_percent_threshold: 0.08
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_task.total_heap_percent_threshold: "0.08"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.heap_variance: 3.0
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_task.heap_variance: "3.0"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.heap_moving_average_window_size: 50
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_task.heap_moving_average_window_size: "50"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_task.cpu_time_millis_threshold: 40000
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_task.cpu_time_millis_threshold: "40000"}
+
+---
+"Test setting search backpressure search_shard_task settings":
+  - skip:
+      version: "- 2.3.99"
+      reason: "Introduced in 2.4.0"
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.elapsed_time_millis_threshold: 50000
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_shard_task.elapsed_time_millis_threshold: "50000"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.heap_percent_threshold: 0.01
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_shard_task.heap_percent_threshold: "0.01"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.total_heap_percent_threshold: 0.08
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_shard_task.total_heap_percent_threshold: "0.08"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.heap_variance: 3.0
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_shard_task.heap_variance: "3.0"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.heap_moving_average_window_size: 50
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_shard_task.heap_moving_average_window_size: "50"}
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search_backpressure.search_shard_task.cpu_time_millis_threshold: 40000
+  - is_true: acknowledged
+
+  - do:
+      cluster.get_settings:
+        flat_settings: false
+  - match: {transient.search_backpressure.search_shard_task.cpu_time_millis_threshold: "40000"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml
index c449a06cf633d..d92538824232d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml
@@ -56,6 +56,12 @@ setup:
         id: 6
         body:
           other_field: "test"
+  - do:
+      index:
+        index: test
+        id: 7
+        body:
+          my_field: "ABCD"
   - do:
       indices.refresh: {}
 
@@ -90,8 +96,9 @@ setup:
           query:
             term:
               my_field.lower: "abcd"
-  - match: { hits.total.value: 1 }
+  - match: { hits.total.value: 2 }
   - match: { hits.hits.0._id: "5" }
+  - match: { hits.hits.1._id: "7" }
 
   - do:
       search:
@@ -100,8 +107,9 @@ setup:
           query:
             term:
               my_field.lower: "ABCD"
-  - match: { hits.total.value: 1 }
+  - match: { hits.total.value: 2 }
   - match: { hits.hits.0._id: "5" }
+  - match: { hits.hits.1._id: "7" }
 
   - do:
       search:
@@ -215,7 +223,7 @@ setup:
             wildcard:
               my_field:
                 value: "*"
-  - match: { hits.total.value: 5 }
+  - match: { hits.total.value: 6 }
 ---
 "regexp match-all works":
   - do:
@@ -226,7 +234,7 @@ setup:
             regexp:
               my_field:
                 value: ".*"
-  - match: { hits.total.value: 5 }
+  - match: { hits.total.value: 6 }
 ---
 "terms query on wildcard field matches":
   - do:
@@ -237,3 +245,28 @@ setup:
             terms: { my_field: ["AbCd"] }
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "5" }
+---
+"case insensitive query on wildcard field":
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            wildcard:
+              my_field:
+                value: "AbCd"
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "5" }
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            wildcard:
+              my_field:
+                value: "AbCd"
+                case_insensitive: true
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._id: "5" }
+  - match: { hits.hits.1._id: "7" }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml
similarity index 99%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml
rename to rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml
index d728070adb188..c885e3fbc2446 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml
@@ -1,7 +1,7 @@
 ---
 setup:
   - skip:
-      version: " - 2.99.99"
+      version: " - 2.16.99"
       reason: The bitmap filtering feature is available in 2.17 and later.
   - do:
       indices.create:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml
new file mode 100644
index 0000000000000..f29fa3f9aa987
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml
@@ -0,0 +1,75 @@
+setup:
+  - do:
+      indices.create:
+        index:  test_1
+        body:
+          mappings:
+            properties:
+              text:
+                type: search_as_you_type
+                fields:
+                  subField:
+                    type: keyword
+  - do:
+      index:
+        index:  test_1
+        id:     1
+        body:   { text: test search as you type }
+
+  - do:
+      indices.refresh:
+        index: [test_1]
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: test_1
+
+# related issue: https://github.com/opensearch-project/OpenSearch/issues/5035
+---
+"Test search_as_you_type data type supports multi-fields":
+  - skip:
+      version: " - 2.17.99"
+      reason: "the bug was fixed since 2.18.0"
+
+  - do:
+      indices.get_mapping: {
+        index: test_1
+      }
+
+  - match: {test_1.mappings.properties.text.type:     search_as_you_type}
+  - match: {test_1.mappings.properties.text.fields.subField.type:     keyword}
+
+  - do:
+      search:
+        index: test_1
+        body:
+          query:
+            multi_match:
+              query: "test search"
+              type: "bool_prefix"
+
+  - match: {hits.total.value: 1}
+
+  - do:
+      search:
+        index: test_1
+        body:
+          query:
+            multi_match:
+              query: "test search"
+              type: "bool_prefix"
+              fields: ["text.subField"]
+
+  - match: {hits.total.value: 1}
+
+  - do:
+      search:
+        index: test_1
+        body:
+          query:
+            term:
+              text.subField: "test search as you type"
+
+  - match: {hits.total.value: 1}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml
index 819f04407b219..c35f2419bdc91 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml
@@ -25,40 +25,38 @@ setup:
         snapshot: test_snapshot
         wait_for_completion: true
 
-# TODO: fix and unmute tests
+  - do:
+      snapshot.status:
+        repository: test_repo_status_1
+        snapshot: test_snapshot
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: test_snapshot }
+  - match: { snapshots.0.state: SUCCESS }
+  - gt: { snapshots.0.stats.incremental.file_count: 0 }
+  - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 }
+  - gt: { snapshots.0.stats.total.file_count: 0 }
+  - gt: { snapshots.0.stats.total.size_in_bytes: 0 }
+  - is_true: snapshots.0.stats.start_time_in_millis
+## fast in memory snapshots can take less than one millisecond to complete.
+  - gte: { snapshots.0.stats.time_in_millis: 0 }
+
+---
+"Get missing snapshot status throws an exception":
+
+  - do:
+      catch: /snapshot_missing_exception.+ is missing/
+      snapshot.status:
+        repository: test_repo_status_1
+        snapshot: test_nonexistent_snapshot
+
+---
+"Get missing snapshot status succeeds when ignoreUnavailable is true":
+
+  - do:
+      snapshot.status:
+        repository: test_repo_status_1
+        snapshot: test_nonexistent_snapshot
+        ignore_unavailable: true
 
-#  - do:
-#      snapshot.status:
-#        repository: test_repo_status_1
-#        snapshot: test_snapshot
-#
-#  - is_true: snapshots
-#  - match: { snapshots.0.snapshot: test_snapshot }
-#  - match: { snapshots.0.state: SUCCESS }
-#  - gt: { snapshots.0.stats.incremental.file_count: 0 }
-#  - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 }
-#  - gt: { snapshots.0.stats.total.file_count: 0 }
-#  - gt: { snapshots.0.stats.total.size_in_bytes: 0 }
-#  - is_true: snapshots.0.stats.start_time_in_millis
-### fast in memory snapshots can take less than one millisecond to complete.
-#  - gte: { snapshots.0.stats.time_in_millis: 0 }
-#
-#---
-#"Get missing snapshot status throws an exception":
-#
-#  - do:
-#      catch: /snapshot_missing_exception.+ is missing/
-#      snapshot.status:
-#        repository: test_repo_status_1
-#        snapshot: test_nonexistent_snapshot
-#
-#---
-#"Get missing snapshot status succeeds when ignoreUnavailable is true":
-#
-#  - do:
-#      snapshot.status:
-#        repository: test_repo_status_1
-#        snapshot: test_nonexistent_snapshot
-#        ignore_unavailable: true
-#
-#  - is_true: snapshots
+  - is_true: snapshots
diff --git a/server/build.gradle b/server/build.gradle
index 0cc42ad690eab..eccaf8a127647 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -127,7 +127,7 @@ dependencies {
   api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}"
 
   // https://mvnrepository.com/artifact/org.roaringbitmap/RoaringBitmap
-  implementation 'org.roaringbitmap:RoaringBitmap:1.2.1'
+  implementation 'org.roaringbitmap:RoaringBitmap:1.3.0'
 
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server
@@ -407,6 +407,8 @@ tasks.named("sourcesJar").configure {
 /** Compares the current build against a laltest released version or the version supplied through 'japicmp.compare.version' system property */
 tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) {
     logger.info("Comparing public APIs from ${version} to ${japicmpCompareTarget}")
+    // See please https://github.com/siom79/japicmp/issues/201
+    compatibilityChangeExcludes = [ "METHOD_ABSTRACT_NOW_DEFAULT" ]
     oldClasspath.from(files("${buildDir}/japicmp-target/opensearch-${japicmpCompareTarget}.jar"))
     newClasspath.from(tasks.named('jar'))
     onlyModified = true
diff --git a/server/licenses/RoaringBitmap-1.2.1.jar.sha1 b/server/licenses/RoaringBitmap-1.2.1.jar.sha1
deleted file mode 100644
index ef8cd48c7a388..0000000000000
--- a/server/licenses/RoaringBitmap-1.2.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-828eb489b5e8c8762f2471010e9c7f20c7de596d
\ No newline at end of file
diff --git a/server/licenses/RoaringBitmap-1.3.0.jar.sha1 b/server/licenses/RoaringBitmap-1.3.0.jar.sha1
new file mode 100644
index 0000000000000..2e48289c4f7af
--- /dev/null
+++ b/server/licenses/RoaringBitmap-1.3.0.jar.sha1
@@ -0,0 +1 @@
+a46ce7a2dc494da69700ab421f081b1583857f6d
\ No newline at end of file
diff --git a/server/licenses/protobuf-java-3.25.4.jar.sha1 b/server/licenses/protobuf-java-3.25.4.jar.sha1
deleted file mode 100644
index 21b6fdfd24dc8..0000000000000
--- a/server/licenses/protobuf-java-3.25.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-43fcb86e4a411516c7fc681450f1516de0b862a2
\ No newline at end of file
diff --git a/server/licenses/protobuf-java-3.25.5.jar.sha1 b/server/licenses/protobuf-java-3.25.5.jar.sha1
new file mode 100644
index 0000000000000..72b42c9efc85a
--- /dev/null
+++ b/server/licenses/protobuf-java-3.25.5.jar.sha1
@@ -0,0 +1 @@
+5ae5c9ec39930ae9b5a61b32b93288818ec05ec1
\ No newline at end of file
diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java
new file mode 100644
index 0000000000000..014e2bf642a4d
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java
@@ -0,0 +1,355 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.cluster.coordination;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.opensearch.cluster.NodeConnectionsService;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.MockEngineFactoryPlugin;
+import org.opensearch.indices.recovery.RecoverySettings;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.tasks.Task;
+import org.opensearch.test.InternalSettingsPlugin;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
+import org.opensearch.test.OpenSearchIntegTestCase.Scope;
+import org.opensearch.test.TestLogsAppender;
+import org.opensearch.test.store.MockFSIndexStore;
+import org.opensearch.test.transport.MockTransportService;
+import org.opensearch.test.transport.StubbableTransport;
+import org.opensearch.transport.ClusterConnectionManager;
+import org.opensearch.transport.TransportChannel;
+import org.opensearch.transport.TransportRequest;
+import org.opensearch.transport.TransportRequestHandler;
+import org.opensearch.transport.TransportService;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.opensearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_ACTION_NAME;
+import static org.hamcrest.Matchers.is;
+
+/**
+ Check https://github.com/opensearch-project/OpenSearch/issues/4874 and
+ https://github.com/opensearch-project/OpenSearch/pull/15521 for context
+ */
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+public class NodeJoinLeftIT extends OpenSearchIntegTestCase {
+
+    private TestLogsAppender testLogsAppender;
+    private String clusterManager;
+    private String redNodeName;
+    private LoggerContext loggerContext;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(
+            MockTransportService.TestPlugin.class,
+            MockFSIndexStore.TestPlugin.class,
+            InternalSettingsPlugin.class,
+            MockEngineFactoryPlugin.class
+        );
+    }
+
+    @Override
+    protected void beforeIndexDeletion() throws Exception {
+        super.beforeIndexDeletion();
+        internalCluster().assertConsistentHistoryBetweenTranslogAndLuceneIndex();
+        internalCluster().assertSeqNos();
+        internalCluster().assertSameDocIdsOnShards();
+    }
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        // Add any other specific messages you want to capture
+        List<String> messagesToCapture = Arrays.asList("failed to join", "IllegalStateException");
+        testLogsAppender = new TestLogsAppender(messagesToCapture);
+        loggerContext = (LoggerContext) LogManager.getContext(false);
+        Configuration config = loggerContext.getConfiguration();
+        LoggerConfig loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName());
+        loggerConfig.addAppender(testLogsAppender, null, null);
+        loggerContext.updateLoggers();
+
+        String indexName = "test";
+        final Settings nodeSettings = Settings.builder()
+            .put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(), "100ms")
+            .put(NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), "10s")
+            .put(FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), "200ms")
+            .put(FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), "100ms")
+            .put(FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), 1)
+            .put(NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), "100ms")
+            .build();
+        // start a 3 node cluster with 1 cluster-manager
+        this.clusterManager = internalCluster().startNode(nodeSettings);
+        internalCluster().startNode(Settings.builder().put("node.attr.color", "blue").put(nodeSettings).build());
+        this.redNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "red").put(nodeSettings).build());
+
+        // validate the 3 node cluster is up
+        ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes(">=3").get();
+        assertThat(response.isTimedOut(), is(false));
+
+        // create an index
+        client().admin()
+            .indices()
+            .prepareCreate(indexName)
+            .setSettings(
+                Settings.builder()
+                    .put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue")
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            )
+            .get();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        testLogsAppender.clearCapturedLogs();
+        loggerContext = (LoggerContext) LogManager.getContext(false);
+        Configuration config = loggerContext.getConfiguration();
+        LoggerConfig loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName());
+        loggerConfig.removeAppender(testLogsAppender.getName());
+        loggerContext.updateLoggers();
+        super.tearDown();
+    }
+
+    public void testClusterStabilityWhenJoinRequestHappensDuringNodeLeftTask() throws Exception {
+
+        ClusterService clusterManagerClsService = internalCluster().getInstance(ClusterService.class, clusterManager);
+        // Simulate a slow applier on the cm to delay node-left state application
+        clusterManagerClsService.addStateApplier(event -> {
+            if (event.nodesRemoved()) {
+                try {
+                    Thread.sleep(3000);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        });
+        // Toggle to succeed/fail the followerchecker to simulate the initial node leaving.
+        AtomicBoolean succeedFollowerChecker = new AtomicBoolean();
+
+        // Simulate followerchecker failure on 1 node when succeedFollowerChecker is false
+        FollowerCheckerBehaviour simulatedFailureBehaviour = new FollowerCheckerBehaviour(() -> {
+            if (succeedFollowerChecker.get()) {
+                return;
+            }
+            try {
+                Thread.sleep(10);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            throw new NodeHealthCheckFailureException("fake followerchecker failure simulated by test to repro race condition");
+        });
+        MockTransportService redTransportService = (MockTransportService) internalCluster().getInstance(
+            TransportService.class,
+            redNodeName
+        );
+        redTransportService.addRequestHandlingBehavior(FOLLOWER_CHECK_ACTION_NAME, simulatedFailureBehaviour);
+
+        // Loop runs 5 times to ensure race condition gets reproduced
+        testLogsAppender.clearCapturedLogs();
+        for (int i = 0; i < 5; i++) {
+            logger.info("--> simulating followerchecker failure to trigger node-left");
+            succeedFollowerChecker.set(false);
+            ClusterHealthResponse response1 = client().admin().cluster().prepareHealth().setWaitForNodes("2").get();
+            assertThat(response1.isTimedOut(), is(false));
+
+            // once we know a node has left, we can re-enable followerchecker to work normally and validate node rejoins
+            logger.info("--> re-enabling normal followerchecker and validating cluster is stable");
+            succeedFollowerChecker.set(true);
+            ClusterHealthResponse response2 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
+            assertThat(response2.isTimedOut(), is(false));
+
+            Thread.sleep(1000);
+            // checking again to validate stability and ensure node did not leave
+            ClusterHealthResponse response3 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
+            assertThat(response3.isTimedOut(), is(false));
+        }
+
+        succeedFollowerChecker.set(true);
+        ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
+        assertThat(response.isTimedOut(), is(false));
+
+        // assert that join requests fail with the right exception
+        boolean logFound = testLogsAppender.waitForLog("failed to join", 30, TimeUnit.SECONDS)
+            && testLogsAppender.waitForLog(
+                "IllegalStateException[cannot make a new connection as disconnect to node",
+                30,
+                TimeUnit.SECONDS
+            );
+        assertTrue("Expected log was not found within the timeout period", logFound);
+    }
+
+    public void testClusterStabilityWhenDisconnectDuringSlowNodeLeftTask() throws Exception {
+        ClusterService clusterManagerClsService = internalCluster().getInstance(ClusterService.class, clusterManager);
+        // Simulate a slow applier on the cm to delay node-left state application
+        clusterManagerClsService.addStateApplier(event -> {
+            if (event.nodesRemoved()) {
+                try {
+                    Thread.sleep(3000);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        });
+        // Toggle to succeed/fail the followerchecker to simulate the initial node leaving.
+        AtomicBoolean succeedFollowerChecker = new AtomicBoolean();
+
+        // Simulate followerchecker failure on 1 node when succeedFollowerChecker is false
+        FollowerCheckerBehaviour simulatedFailureBehaviour = new FollowerCheckerBehaviour(() -> {
+            if (succeedFollowerChecker.get()) {
+                return;
+            }
+            try {
+                Thread.sleep(10);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            throw new NodeHealthCheckFailureException("fake followerchecker failure simulated by test to repro race condition");
+        });
+        MockTransportService cmTransportService = (MockTransportService) internalCluster().getInstance(
+            TransportService.class,
+            clusterManager
+        );
+        MockTransportService redTransportService = (MockTransportService) internalCluster().getInstance(
+            TransportService.class,
+            redNodeName
+        );
+        redTransportService.addRequestHandlingBehavior(FOLLOWER_CHECK_ACTION_NAME, simulatedFailureBehaviour);
+
+        // Loop runs 5 times to ensure race condition gets reproduced
+        testLogsAppender.clearCapturedLogs();
+        for (int i = 0; i < 5; i++) {
+            // Fail followerchecker by force to trigger node disconnect and node left
+            logger.info("--> simulating followerchecker failure to trigger node-left");
+            succeedFollowerChecker.set(false);
+            Thread.sleep(1000);
+
+            // Trigger a node disconnect while node-left task is still processing
+            logger.info(
+                "--> triggering a simulated disconnect on red node, after the follower checker failed to see how node-left task deals with this"
+            );
+            cmTransportService.disconnectFromNode(redTransportService.getLocalDiscoNode());
+
+            ClusterHealthResponse response1 = client().admin().cluster().prepareHealth().setWaitForNodes("2").get();
+            assertThat(response1.isTimedOut(), is(false));
+
+            // once we know a node has left, we can re-enable followerchecker to work normally and validate node rejoins
+            logger.info("--> re-enabling normal followerchecker and validating cluster is stable");
+            succeedFollowerChecker.set(true);
+            ClusterHealthResponse response2 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
+            assertThat(response2.isTimedOut(), is(false));
+
+            Thread.sleep(1000);
+            // checking again to validate stability and ensure node did not leave
+            ClusterHealthResponse response3 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
+            assertThat(response3.isTimedOut(), is(false));
+        }
+
+        succeedFollowerChecker.set(true);
+        ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
+        assertThat(response.isTimedOut(), is(false));
+
+        // assert that join requests fail with the right exception
+        boolean logFound = testLogsAppender.waitForLog("failed to join", 30, TimeUnit.SECONDS);
+        assertTrue("Expected log was not found within the timeout period", logFound);
+        logFound = testLogsAppender.waitForLog(
+            "IllegalStateException[cannot make a new connection as disconnect to node",
+            30,
+            TimeUnit.SECONDS
+        );
+        assertTrue("Expected log was not found within the timeout period", logFound);
+    }
+
+    public void testRestartDataNode() throws Exception {
+
+        Settings redNodeDataPathSettings = internalCluster().dataPathSettings(redNodeName);
+        logger.info("-> stopping data node");
+        internalCluster().stopRandomNode(settings -> settings.get("node.name").equals(redNodeName));
+        ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("2").get();
+        assertThat(response.isTimedOut(), is(false));
+
+        logger.info("-> restarting stopped node");
+        internalCluster().startNode(Settings.builder().put("node.name", redNodeName).put(redNodeDataPathSettings).build());
+        response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
+        assertThat(response.isTimedOut(), is(false));
+    }
+
+    public void testRestartCmNode() throws Exception {
+
+        Settings cmNodeSettings = internalCluster().dataPathSettings(clusterManager);
+
+        logger.info("-> stopping cluster-manager node");
+        internalCluster().stopRandomNode(settings -> settings.get("node.name").equals(clusterManager));
+        ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("2").get();
+        assertThat(response.isTimedOut(), is(false));
+
+        logger.info("-> restarting stopped node");
+        internalCluster().startNode(Settings.builder().put("node.name", clusterManager).put(cmNodeSettings).build());
+        response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
+        assertThat(response.isTimedOut(), is(false));
+    }
+
+    private class FollowerCheckerBehaviour implements StubbableTransport.RequestHandlingBehavior<TransportRequest> {
+        private final Runnable connectionBreaker;
+
+        private FollowerCheckerBehaviour(Runnable connectionBreaker) {
+            this.connectionBreaker = connectionBreaker;
+        }
+
+        @Override
+        public void messageReceived(
+            TransportRequestHandler<TransportRequest> handler,
+            TransportRequest request,
+            TransportChannel channel,
+            Task task
+        ) throws Exception {
+
+            connectionBreaker.run();
+            handler.messageReceived(request, channel, task);
+        }
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java
new file mode 100644
index 0000000000000..98859f517aad4
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java
@@ -0,0 +1,299 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway.remote;
+
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.remotemigration.MigrationBaseTestCase;
+import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
+import org.opensearch.repositories.fs.ReloadableFsRepository;
+import org.opensearch.test.InternalSettingsPlugin;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.transport.MockTransportService;
+import org.junit.Assert;
+import org.junit.Before;
+
+import java.util.Collection;
+import java.util.Locale;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+/**
+ * Tests the compatibility between types of nodes based on the configured repositories
+ *  Non Remote node [No Repositories configured]
+ *  Remote Publish Configured Node [Cluster State + Routing Table]
+ *  Remote Node [Cluster State + Segment + Translog]
+ *  Remote Node With Routing Table [Cluster State + Segment + Translog + Routing Table]
+ */
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RemotePublicationConfigurationIT extends MigrationBaseTestCase {
+    private final String REMOTE_PRI_DOCREP_REP = "remote-primary-docrep-replica";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        /* Adding the following mock plugins:
+        - InternalSettingsPlugin : To override default intervals of retention lease and global ckp sync
+        - MockFsRepositoryPlugin and MockTransportService.TestPlugin: To ensure remote interactions are not no-op and retention leases are properly propagated
+         */
+        return Stream.concat(
+            super.nodePlugins().stream(),
+            Stream.of(InternalSettingsPlugin.class, MockFsRepositoryPlugin.class, MockTransportService.TestPlugin.class)
+        ).collect(Collectors.toList());
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        if (segmentRepoPath == null || translogRepoPath == null) {
+            segmentRepoPath = randomRepoPath().toAbsolutePath();
+            translogRepoPath = randomRepoPath().toAbsolutePath();
+        }
+        super.setUp();
+    }
+
+    public Settings.Builder remotePublishConfiguredNodeSetting() {
+        String stateRepoSettingsAttributeKeyPrefix = String.format(
+            Locale.getDefault(),
+            "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
+            REPOSITORY_NAME
+        );
+        String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey();
+        String stateRepoTypeAttributeKey = String.format(
+            Locale.getDefault(),
+            "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
+            REPOSITORY_NAME
+        );
+        String routingTableRepoTypeAttributeKey = String.format(
+            Locale.getDefault(),
+            "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
+            ROUTING_TABLE_REPO_NAME
+        );
+        String routingTableRepoSettingsAttributeKeyPrefix = String.format(
+            Locale.getDefault(),
+            "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
+            ROUTING_TABLE_REPO_NAME
+        );
+
+        Settings.Builder builder = Settings.builder()
+            .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)
+            .put(stateRepoTypeAttributeKey, ReloadableFsRepository.TYPE)
+            .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
+            .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, true)
+            .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
+            .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO_NAME)
+            .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE)
+            .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath);
+        return builder;
+    }
+
+    public Settings.Builder remoteWithRoutingTableNodeSetting() {
+        // Remote Cluster with Routing table
+        return Settings.builder()
+            .put(
+                buildRemoteStoreNodeAttributes(
+                    REPOSITORY_NAME,
+                    segmentRepoPath,
+                    REPOSITORY_2_NAME,
+                    translogRepoPath,
+                    REPOSITORY_NAME,
+                    segmentRepoPath,
+                    false
+                )
+            )
+            .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true);
+    }
+
+    public void testRemoteClusterStateServiceNotInitialized_WhenNodeAttributesNotPresent() {
+        internalCluster().startClusterManagerOnlyNode();
+        internalCluster().startDataOnlyNodes(2);
+
+        ensureStableCluster(3);
+        ensureGreen();
+
+        internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNull);
+    }
+
+    public void testServiceInitialized_WhenNodeAttributesPresent() {
+        internalCluster().startClusterManagerOnlyNode(
+            buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE)
+        );
+        internalCluster().startDataOnlyNodes(
+            2,
+            buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE)
+        );
+
+        ensureStableCluster(3);
+        ensureGreen();
+
+        internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNotNull);
+    }
+
+    public void testRemotePublishConfigNodeJoinNonRemoteCluster() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+        internalCluster().startDataOnlyNodes(2);
+
+        Settings.Builder build = remotePublishConfiguredNodeSetting();
+        internalCluster().startClusterManagerOnlyNode(build.build());
+        internalCluster().startDataOnlyNodes(2, build.build());
+
+        ensureStableCluster(6);
+        ensureGreen();
+    }
+
+    public void testRemotePublishConfigNodeJoinRemoteCluster() throws Exception {
+        // Remote Cluster without Routing table
+        setAddRemote(true);
+        internalCluster().startClusterManagerOnlyNode();
+        internalCluster().startDataOnlyNodes(2);
+        setAddRemote(false);
+
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(
+            Settings.builder()
+                .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
+                .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
+        );
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+        Settings.Builder build = remotePublishConfiguredNodeSetting();
+        internalCluster().startClusterManagerOnlyNode(build.build());
+        ensureStableCluster(4);
+        ensureGreen();
+    }
+
+    public void testRemoteNodeWithRoutingTableJoinRemoteCluster() throws Exception {
+        setAddRemote(true);
+        internalCluster().startClusterManagerOnlyNode();
+        internalCluster().startDataOnlyNodes(2);
+        setAddRemote(false);
+
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(
+            Settings.builder()
+                .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
+                .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
+        );
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        // Remote Repo with Routing table
+        Settings settings = remoteWithRoutingTableNodeSetting().build();
+        internalCluster().startClusterManagerOnlyNode(settings);
+        ensureStableCluster(4);
+        ensureGreen();
+    }
+
+    public void testNonRemoteNodeJoinRemoteWithRoutingCluster() throws Exception {
+        Settings settings = remoteWithRoutingTableNodeSetting().build();
+        internalCluster().startClusterManagerOnlyNode(settings);
+        internalCluster().startDataOnlyNodes(2, settings);
+
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(
+            Settings.builder()
+                .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
+                .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
+        );
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        internalCluster().startClusterManagerOnlyNode();
+        ensureStableCluster(4);
+        ensureGreen();
+    }
+
+    public void testRemotePublishConfigNodeJoinRemoteWithRoutingCluster() throws Exception {
+        Settings settings = remoteWithRoutingTableNodeSetting().build();
+        internalCluster().startClusterManagerOnlyNode(settings);
+        internalCluster().startDataOnlyNodes(2, settings);
+
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(
+            Settings.builder()
+                .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
+                .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
+        );
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        internalCluster().startClusterManagerOnlyNode(remotePublishConfiguredNodeSetting().build());
+
+        ensureStableCluster(4);
+        ensureGreen();
+    }
+
+    public void testNonRemoteNodeJoiningPublishConfigCluster() throws Exception {
+        Settings.Builder build = remotePublishConfiguredNodeSetting();
+        internalCluster().startClusterManagerOnlyNode(build.build());
+        internalCluster().startDataOnlyNodes(2, build.build());
+
+        internalCluster().startClusterManagerOnlyNode();
+
+        ensureStableCluster(4);
+        ensureGreen();
+    }
+
+    public void testRemoteNodeJoiningPublishConfigCluster() throws Exception {
+        Settings.Builder build = remotePublishConfiguredNodeSetting();
+        internalCluster().startClusterManagerOnlyNode(build.build());
+        internalCluster().startDataOnlyNodes(2, build.build());
+
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(
+            Settings.builder()
+                .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
+                .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
+        );
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        setAddRemote(true);
+        internalCluster().startClusterManagerOnlyNode();
+        ensureStableCluster(4);
+        ensureGreen();
+    }
+
+    public void testRemoteNodeWithRoutingTableJoiningPublishConfigCluster() throws Exception {
+        Settings.Builder build = remotePublishConfiguredNodeSetting();
+        internalCluster().startClusterManagerOnlyNode(build.build());
+        internalCluster().startDataOnlyNodes(2, build.build());
+
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(
+            Settings.builder()
+                .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
+                .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
+        );
+
+        Settings settings = Settings.builder()
+            .put(
+                buildRemoteStoreNodeAttributes(
+                    REPOSITORY_NAME,
+                    segmentRepoPath,
+                    REPOSITORY_2_NAME,
+                    translogRepoPath,
+                    ROUTING_TABLE_REPO_NAME,
+                    segmentRepoPath,
+                    false
+                )
+            )
+            .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
+            .build();
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+        internalCluster().startClusterManagerOnlyNode(settings);
+
+        ensureStableCluster(4);
+        ensureGreen();
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java
index faab3645ae894..2409b5d0d0e45 100644
--- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java
@@ -56,6 +56,7 @@
 import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY;
 import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
+import static org.opensearch.gateway.remote.RemoteDownloadStats.CHECKSUM_VALIDATION_FAILED_COUNT;
 import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS;
 import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA;
@@ -214,7 +215,11 @@ public void testRemotePublicationDisableIfRemoteStateDisabled() {
         ensureStableCluster(5);
         ensureGreen(INDEX_NAME);
 
-        assertNull(internalCluster().getCurrentClusterManagerNodeInstance(RemoteClusterStateService.class));
+        RemoteClusterStateService remoteClusterStateService = internalCluster().getCurrentClusterManagerNodeInstance(
+            RemoteClusterStateService.class
+        );
+
+        assertFalse(remoteClusterStateService.isRemotePublicationEnabled());
     }
 
     public void testRemotePublicationDownloadStats() {
@@ -234,7 +239,6 @@ public void testRemotePublicationDownloadStats() {
         assertDataNodeDownloadStats(nodesStatsResponseDataNode);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15767")
     public void testRemotePublicationDisabledByRollingRestart() throws Exception {
         prepareCluster(3, 2, INDEX_NAME, 1, 2);
         ensureStableCluster(5);
@@ -271,7 +275,6 @@ public void doAfterNodes(int n, Client client) {
                             assertTrue(
                                 stats.getFullClusterStateReceivedCount() > 0 || stats.getCompatibleClusterStateDiffReceivedCount() > 0
                             );
-                            assertEquals(0, stats.getIncompatibleClusterStateDiffReceivedCount());
                         } else {
                             DiscoveryStats stats = nodeStats.getDiscoveryStats();
                             assertEquals(0, stats.getPublishStats().getFullClusterStateReceivedCount());
@@ -296,7 +299,7 @@ public void doAfterNodes(int n, Client client) {
                         );
                         if (activeCMRestarted) {
                             assertNull(remoteState.getLastAcceptedState());
-                            // assertNull(remoteState.getLastAcceptedManifest());
+                            assertNull(remoteState.getLastAcceptedManifest());
                         } else {
                             ClusterState localState = registry.getPersistedState(PersistedStateRegistry.PersistedStateType.LOCAL)
                                 .getLastAcceptedState();
@@ -325,7 +328,6 @@ public void doAfterNodes(int n, Client client) {
         response.getNodes().forEach(nodeStats -> {
             PublishClusterStateStats stats = nodeStats.getDiscoveryStats().getPublishStats();
             assertTrue(stats.getFullClusterStateReceivedCount() > 0 || stats.getCompatibleClusterStateDiffReceivedCount() > 0);
-            assertEquals(0, stats.getIncompatibleClusterStateDiffReceivedCount());
         });
         NodesInfoResponse nodesInfoResponse = client().admin()
             .cluster()
@@ -340,7 +342,7 @@ public void doAfterNodes(int n, Client client) {
             PersistedStateRegistry registry = internalCluster().getInstance(PersistedStateRegistry.class, node);
             CoordinationState.PersistedState remoteState = registry.getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE);
             assertNull(remoteState.getLastAcceptedState());
-            // assertNull(remoteState.getLastAcceptedManifest());
+            assertNull(remoteState.getLastAcceptedManifest());
         });
     }
 
@@ -405,10 +407,28 @@ private void assertDataNodeDownloadStats(NodesStatsResponse nodesStatsResponse)
         assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getSuccessCount() > 0);
         assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getFailedCount());
         assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getTotalTimeInMillis() > 0);
+        assertEquals(
+            0,
+            dataNodeDiscoveryStats.getClusterStateStats()
+                .getPersistenceStats()
+                .get(0)
+                .getExtendedFields()
+                .get(CHECKSUM_VALIDATION_FAILED_COUNT)
+                .get()
+        );
 
         assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getSuccessCount() > 0);
         assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getFailedCount());
         assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getTotalTimeInMillis() > 0);
+        assertEquals(
+            0,
+            dataNodeDiscoveryStats.getClusterStateStats()
+                .getPersistenceStats()
+                .get(1)
+                .getExtendedFields()
+                .get(CHECKSUM_VALIDATION_FAILED_COUNT)
+                .get()
+        );
     }
 
     private Map<String, Integer> getMetadataFiles(BlobStoreRepository repository, String subDirectory) throws IOException {
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java
index 17a9c3ddbe317..a82e6d45ce0f6 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java
@@ -57,6 +57,8 @@
 
 public class MigrationBaseTestCase extends OpenSearchIntegTestCase {
     protected static final String REPOSITORY_NAME = "test-remote-store-repo";
+    protected static final String ROUTING_TABLE_REPO_NAME = "remote-routing-repo";
+
     protected static final String REPOSITORY_2_NAME = "test-remote-store-repo-2";
 
     protected Path segmentRepoPath;
@@ -72,7 +74,7 @@ public class MigrationBaseTestCase extends OpenSearchIntegTestCase {
         randomAlphaOfLength(5)
     );
 
-    void setAddRemote(boolean addRemote) {
+    public void setAddRemote(boolean addRemote) {
         this.addRemote = addRemote;
     }
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
index b55219e1cb37f..35689deeb8134 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
@@ -19,7 +19,6 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.util.FileSystemUtils;
 import org.opensearch.index.remote.RemoteIndexPath;
-import org.opensearch.index.remote.RemoteIndexPathUploader;
 import org.opensearch.index.remote.RemoteStoreEnums;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.test.InternalTestCluster;
@@ -533,17 +532,18 @@ public void testRemoteIndexPathFileExistsAfterMigration() throws Exception {
 
         // validate remote index path file exists
         logger.info("---> Asserting remote index path file exists");
-        String fileNamePrefix = String.join(RemoteIndexPathUploader.DELIMITER, indexUUID, "7", RemoteIndexPath.DEFAULT_VERSION);
 
         assertTrue(FileSystemUtils.exists(translogRepoPath.resolve(RemoteIndexPath.DIR)));
         Path[] files = FileSystemUtils.files(translogRepoPath.resolve(RemoteIndexPath.DIR));
         assertEquals(1, files.length);
-        assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(fileNamePrefix)));
+        logger.info(files[0].toString());
+        assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(indexUUID)));
 
         assertTrue(FileSystemUtils.exists(segmentRepoPath.resolve(RemoteIndexPath.DIR)));
         files = FileSystemUtils.files(segmentRepoPath.resolve(RemoteIndexPath.DIR));
         assertEquals(1, files.length);
-        assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(fileNamePrefix)));
+        logger.info(files[0].toString());
+        assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(indexUUID)));
     }
 
     /**
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
index 30c597e405f4e..d9e72dd137182 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
@@ -12,6 +12,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
@@ -68,7 +69,6 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode()
         assertRemoteStoreBackedIndex(indexName2);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15793")
     public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() throws Exception {
         logger.info("Initialize cluster: gives non remote cluster manager");
         initializeCluster(false);
@@ -76,7 +76,10 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix
         logger.info("Add remote and non-remote nodes");
         setClusterMode(MIXED.mode);
         addRemote = false;
-        String nonRemoteNodeName = internalCluster().startNode();
+        Settings settings = Settings.builder()
+            .put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), snapshotShardPathFixedPrefix ? "c" : "")
+            .build();
+        String nonRemoteNodeName = internalCluster().startNode(settings);
         addRemote = true;
         String remoteNodeName = internalCluster().startNode();
         internalCluster().validateClusterFormed();
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
index 0acb578e2e7bf..927dbf9995778 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
@@ -9,8 +9,11 @@
 package org.opensearch.remotestore;
 
 import org.opensearch.action.DocWriteResponse;
+import org.opensearch.action.LatchedActionListener;
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
 import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.action.admin.indices.recovery.RecoveryResponse;
@@ -25,7 +28,9 @@
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.io.IOUtils;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.rest.RestStatus;
@@ -38,19 +43,17 @@
 import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.recovery.RecoveryState;
 import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.Repository;
 import org.opensearch.repositories.RepositoryData;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.repositories.fs.FsRepository;
-import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.opensearch.snapshots.SnapshotInfo;
 import org.opensearch.snapshots.SnapshotRestoreException;
 import org.opensearch.snapshots.SnapshotState;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
-import org.junit.After;
-import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -62,7 +65,10 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -72,6 +78,7 @@
 import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
 import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
 import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
+import static org.opensearch.snapshots.SnapshotsService.getPinningEntity;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -79,48 +86,7 @@
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
-public class RemoteRestoreSnapshotIT extends AbstractSnapshotIntegTestCase {
-    private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX;
-    private Path remoteRepoPath;
-
-    @Before
-    public void setup() {
-        remoteRepoPath = randomRepoPath().toAbsolutePath();
-    }
-
-    @After
-    public void teardown() {
-        clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get();
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal))
-            .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath))
-            .build();
-    }
-
-    private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) {
-        Settings.Builder settingsBuilder = Settings.builder()
-            .put(super.indexSettings())
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
-            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s");
-        return settingsBuilder;
-    }
-
-    private void indexDocuments(Client client, String indexName, int numOfDocs) {
-        indexDocuments(client, indexName, 0, numOfDocs);
-    }
-
-    private void indexDocuments(Client client, String indexName, int fromId, int toId) {
-        for (int i = fromId; i < toId; i++) {
-            String id = Integer.toString(i);
-            client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get();
-        }
-        client.admin().indices().prepareFlush(indexName).get();
-    }
+public class RemoteRestoreSnapshotIT extends RemoteSnapshotIT {
 
     private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) {
         for (int i = 0; i < numOfDocs; i++) {
@@ -793,17 +759,15 @@ public void testInvalidRestoreRequestScenarios() throws Exception {
         assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore"));
     }
 
-    public void testCreateSnapshotV2() throws Exception {
+    public void testCreateSnapshotV2_Orphan_Timestamp_Cleanup() throws Exception {
         internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
         internalCluster().startDataOnlyNode(pinnedTimestampSettings());
         internalCluster().startDataOnlyNode(pinnedTimestampSettings());
         String indexName1 = "testindex1";
         String indexName2 = "testindex2";
-        String indexName3 = "testindex3";
         String snapshotRepoName = "test-create-snapshot-repo";
         String snapshotName1 = "test-create-snapshot1";
         Path absolutePath1 = randomRepoPath().toAbsolutePath();
-        logger.info("Snapshot Path [{}]", absolutePath1);
 
         Settings.Builder settings = Settings.builder()
             .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
@@ -827,27 +791,37 @@ public void testCreateSnapshotV2() throws Exception {
         indexDocuments(client, indexName2, numDocsInIndex2);
         ensureGreen(indexName1, indexName2);
 
+        // create an orphan timestamp related to this repo
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            internalCluster().getClusterManagerName()
+        );
+        forceSyncPinnedTimestamps();
+
+        long pinnedTimestamp = System.currentTimeMillis();
+        final CountDownLatch latch = new CountDownLatch(1);
+        LatchedActionListener<Void> latchedActionListener = new LatchedActionListener<>(new ActionListener<>() {
+            @Override
+            public void onResponse(Void unused) {}
+
+            @Override
+            public void onFailure(Exception e) {}
+        }, latch);
+
+        remoteStorePinnedTimestampService.pinTimestamp(
+            pinnedTimestamp,
+            getPinningEntity(snapshotRepoName, "some_uuid"),
+            latchedActionListener
+        );
+        latch.await();
+
         SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, Collections.emptyList());
         assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
         assertThat(snapshotInfo.successfulShards(), greaterThan(0));
         assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
         assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L));
 
-        indexDocuments(client, indexName1, 10);
-        indexDocuments(client, indexName2, 20);
-
-        createIndex(indexName3, indexSettings);
-        indexDocuments(client, indexName3, 10);
-
-        String snapshotName2 = "test-create-snapshot2";
-
-        // verify response status if waitForCompletion is not true
-        RestStatus createSnapshotResponseStatus = client().admin()
-            .cluster()
-            .prepareCreateSnapshot(snapshotRepoName, snapshotName2)
-            .get()
-            .status();
-        assertEquals(RestStatus.ACCEPTED, createSnapshotResponseStatus);
+        waitUntil(() -> 1 == RemoteStorePinnedTimestampService.getPinnedEntities().size());
     }
 
     public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception {
@@ -919,7 +893,8 @@ public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception
         assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
         assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2));
         assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L));
-
+        forceSyncPinnedTimestamps();
+        assertEquals(RemoteStorePinnedTimestampService.getPinnedEntities().size(), 1);
     }
 
     public void testConcurrentSnapshotV2CreateOperation() throws InterruptedException, ExecutionException {
@@ -995,6 +970,156 @@ public void testConcurrentSnapshotV2CreateOperation() throws InterruptedExceptio
 
         RepositoryData repositoryData = repositoryDataPlainActionFuture.get();
         assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1));
+        forceSyncPinnedTimestamps();
+        assertEquals(RemoteStorePinnedTimestampService.getPinnedEntities().size(), repositoryData.getSnapshotIds().size());
+    }
+
+    public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Exception {
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String snapshotRepoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(snapshotRepoName, FsRepository.TYPE, settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        Thread thread = new Thread(() -> {
+            try {
+                String snapshotName = "snapshot-earlier-master";
+                internalCluster().nonClusterManagerClient()
+                    .admin()
+                    .cluster()
+                    .prepareCreateSnapshot(snapshotRepoName, snapshotName)
+                    .setWaitForCompletion(true)
+                    .setMasterNodeTimeout(TimeValue.timeValueSeconds(60))
+                    .get();
+
+            } catch (Exception ignored) {}
+        });
+        thread.start();
+
+        // stop existing master
+        final String clusterManagerNode = internalCluster().getClusterManagerName();
+        stopNode(clusterManagerNode);
+
+        // Validate that we have greater one snapshot has been created
+        String snapshotName = "new-snapshot";
+        try {
+            client().admin().cluster().prepareCreateSnapshot(snapshotRepoName, snapshotName).setWaitForCompletion(true).get();
+        } catch (Exception e) {
+            logger.info("Exception while creating new-snapshot", e);
+        }
+
+        AtomicLong totalSnaps = new AtomicLong();
+
+        // Validate that snapshot is present in repository data
+        assertBusy(() -> {
+            GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName);
+            GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet();
+            assertThat(response2.getSnapshots().size(), greaterThanOrEqualTo(1));
+            totalSnaps.set(response2.getSnapshots().size());
+
+        }, 30, TimeUnit.SECONDS);
+        thread.join();
+        forceSyncPinnedTimestamps();
+        waitUntil(() -> {
+            this.forceSyncPinnedTimestamps();
+            return RemoteStorePinnedTimestampService.getPinnedEntities().size() == totalSnaps.intValue();
+        });
+    }
+
+    public void testCreateSnapshotV2() throws Exception {
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String indexName3 = "testindex3";
+        String snapshotRepoName = "test-create-snapshot-repo";
+        String snapshotName1 = "test-create-snapshot1";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+
+        createRepository(snapshotRepoName, FsRepository.TYPE, settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, Collections.emptyList());
+        assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(snapshotInfo.successfulShards(), greaterThan(0));
+        assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
+        assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L));
+
+        indexDocuments(client, indexName1, 10);
+        indexDocuments(client, indexName2, 20);
+
+        createIndex(indexName3, indexSettings);
+        indexDocuments(client, indexName3, 10);
+
+        String snapshotName2 = "test-create-snapshot2";
+
+        // verify response status if waitForCompletion is not true
+        RestStatus createSnapshotResponseStatus = client().admin()
+            .cluster()
+            .prepareCreateSnapshot(snapshotRepoName, snapshotName2)
+            .get()
+            .status();
+        assertEquals(RestStatus.ACCEPTED, createSnapshotResponseStatus);
+        forceSyncPinnedTimestamps();
+        assertEquals(2, RemoteStorePinnedTimestampService.getPinnedEntities().size());
+    }
+
+    public void forceSyncPinnedTimestamps() {
+        // for all nodes , run forceSyncPinnedTimestamps()
+        for (String node : internalCluster().getNodeNames()) {
+            RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+                RemoteStorePinnedTimestampService.class,
+                node
+            );
+            remoteStorePinnedTimestampService.forceSyncPinnedTimestamps();
+        }
     }
 
     public void testCreateSnapshotV2WithRedIndex() throws Exception {
@@ -1315,11 +1440,4 @@ public void testConcurrentV1SnapshotAndV2RepoSettingUpdate() throws Exception {
         createV1SnapshotThread.join();
     }
 
-    private Settings pinnedTimestampSettings() {
-        Settings settings = Settings.builder()
-            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
-            .build();
-        return settings;
-    }
-
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java
new file mode 100644
index 0000000000000..836871b8251d1
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java
@@ -0,0 +1,89 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotestore;
+
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
+import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.RepositoryMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.indices.RemoteStoreSettings;
+import org.opensearch.repositories.fs.ReloadableFsRepository;
+import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.nio.file.Path;
+import java.util.concurrent.ExecutionException;
+
+import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING;
+
+public abstract class RemoteSnapshotIT extends AbstractSnapshotIntegTestCase {
+    protected static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX;
+    protected Path remoteRepoPath;
+
+    @Before
+    public void setup() {
+        remoteRepoPath = randomRepoPath().toAbsolutePath();
+    }
+
+    @After
+    public void teardown() {
+        clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get();
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath))
+            .build();
+    }
+
+    protected Settings pinnedTimestampSettings() {
+        Settings settings = Settings.builder()
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
+            .build();
+        return settings;
+    }
+
+    protected Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) {
+        Settings.Builder settingsBuilder = Settings.builder()
+            .put(super.indexSettings())
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s");
+        return settingsBuilder;
+    }
+
+    protected void indexDocuments(Client client, String indexName, int numOfDocs) {
+        indexDocuments(client, indexName, 0, numOfDocs);
+    }
+
+    void indexDocuments(Client client, String indexName, int fromId, int toId) {
+        for (int i = fromId; i < toId; i++) {
+            String id = Integer.toString(i);
+            client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get();
+        }
+        client.admin().indices().prepareFlush(indexName).get();
+    }
+
+    protected void setFailRate(String repoName, int value) throws ExecutionException, InterruptedException {
+        GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { repoName });
+        GetRepositoriesResponse res = client().admin().cluster().getRepositories(gr).get();
+        RepositoryMetadata rmd = res.repositories().get(0);
+        Settings.Builder settings = Settings.builder()
+            .put("location", rmd.settings().get("location"))
+            .put(REPOSITORIES_FAILRATE_SETTING.getKey(), value);
+        createRepository(repoName, ReloadableFsRepository.TYPE, settings);
+    }
+
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
index d078ba05faa12..6a5adf5ea4fb7 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -42,6 +42,7 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.junit.Before;
 
+import java.io.IOError;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -339,10 +340,11 @@ public void testFullClusterRestoreManifestFilePointsToInvalidIndexMetadataPathTh
             for (UploadedIndexMetadata md : manifest.getIndices()) {
                 Files.move(segmentRepoPath.resolve(md.getUploadedFilename()), segmentRepoPath.resolve("cluster-state/"));
             }
+            internalCluster().stopAllNodes();
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-        assertThrows(IllegalStateException.class, () -> addNewNodes(dataNodeCount, clusterManagerNodeCount));
+        assertThrows(IOError.class, () -> internalCluster().client());
         // Test is complete
 
         // Starting a node without remote state to ensure test cleanup
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
index 692727357a88a..ebb911c739eb3 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
@@ -18,6 +18,7 @@
 import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.action.search.SearchPhaseExecutionException;
+import org.opensearch.client.Requests;
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.routing.RecoverySource;
@@ -202,7 +203,7 @@ public void testRemoteTranslogCleanup() throws Exception {
 
     public void testStaleCommitDeletionWithInvokeFlush() throws Exception {
         String dataNode = internalCluster().startNode();
-        createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1));
+        createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000L, -1));
         int numberOfIterations = randomIntBetween(5, 15);
         indexData(numberOfIterations, true, INDEX_NAME);
         String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings());
@@ -1011,4 +1012,70 @@ public void testAsyncTranslogDurabilityRestrictionsThroughIdxTemplates() throws
                 .get()
         );
     }
+
+    public void testCloseIndexWithNoOpSyncAndFlushForSyncTranslog() throws InterruptedException {
+        internalCluster().startNodes(3);
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "5s"))
+            .get();
+        Settings.Builder settings = Settings.builder()
+            .put(remoteStoreIndexSettings(0, 10000L, -1))
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "1s");
+        createIndex(INDEX_NAME, settings.build());
+        CountDownLatch latch = new CountDownLatch(1);
+        new Thread(() -> {
+            if (randomBoolean()) {
+                for (int i = 0; i < randomIntBetween(1, 5); i++) {
+                    indexSingleDoc(INDEX_NAME);
+                }
+                flushAndRefresh(INDEX_NAME);
+            }
+            // Index single doc to start the asyn io processor to run which will lead to 10s wait time before the next sync.
+            indexSingleDoc(INDEX_NAME);
+            // Reduce the latch for the main thread to flush after some sleep.
+            latch.countDown();
+            // Index another doc and in this case the flush would have happened before the sync.
+            indexSingleDoc(INDEX_NAME);
+        }).start();
+        // Wait for atleast one doc to be ingested.
+        latch.await();
+        // Sleep for some time for the next doc to be present in lucene buffer. If flush happens first before the doc #2
+        // gets indexed, then it goes into the happy case where the close index happens succefully.
+        Thread.sleep(1000);
+        // Flush so that the subsequent sync or flushes are no-op.
+        flush(INDEX_NAME);
+        // Closing the index involves translog.sync and shard.flush which are now no-op.
+        client().admin().indices().close(Requests.closeIndexRequest(INDEX_NAME)).actionGet();
+        Thread.sleep(10000);
+        ensureGreen(INDEX_NAME);
+    }
+
+    public void testCloseIndexWithNoOpSyncAndFlushForAsyncTranslog() throws InterruptedException {
+        internalCluster().startNodes(3);
+        Settings.Builder settings = Settings.builder()
+            .put(remoteStoreIndexSettings(0, 10000L, -1))
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "1s")
+            .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Durability.ASYNC)
+            .put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "10s");
+        createIndex(INDEX_NAME, settings.build());
+        CountDownLatch latch = new CountDownLatch(1);
+        new Thread(() -> {
+            // Index some docs to start the asyn io processor to run which will lead to 10s wait time before the next sync.
+            indexSingleDoc(INDEX_NAME);
+            indexSingleDoc(INDEX_NAME);
+            indexSingleDoc(INDEX_NAME);
+            // Reduce the latch for the main thread to flush after some sleep.
+            latch.countDown();
+        }).start();
+        // Wait for atleast one doc to be ingested.
+        latch.await();
+        // Flush so that the subsequent sync or flushes are no-op.
+        flush(INDEX_NAME);
+        // Closing the index involves translog.sync and shard.flush which are now no-op.
+        client().admin().indices().close(Requests.closeIndexRequest(INDEX_NAME)).actionGet();
+        Thread.sleep(10000);
+        ensureGreen(INDEX_NAME);
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java
new file mode 100644
index 0000000000000..08ece7df457cc
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java
@@ -0,0 +1,508 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotestore;
+
+import org.opensearch.action.support.IndicesOptions;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.index.translog.transfer.TranslogTransferMetadata;
+import org.opensearch.indices.RemoteStoreSettings;
+import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+
+import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RemoteStorePinnedTimestampsGarbageCollectionIT extends RemoteStoreBaseIntegTestCase {
+    static final String INDEX_NAME = "remote-store-test-idx-1";
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), false)
+            .build();
+    }
+
+    private void keepPinnedTimestampSchedulerUpdated() throws InterruptedException {
+        long currentTime = System.currentTimeMillis();
+        int maxRetry = 10;
+        while (maxRetry > 0 && RemoteStorePinnedTimestampService.getPinnedTimestamps().v1() <= currentTime) {
+            Thread.sleep(1000);
+            maxRetry--;
+        }
+    }
+
+    ActionListener<Void> noOpActionListener = new ActionListener<>() {
+        @Override
+        public void onResponse(Void unused) {}
+
+        @Override
+        public void onFailure(Exception e) {}
+    };
+
+    public void testLiveIndexNoPinnedTimestamps() throws Exception {
+        prepareCluster(1, 1, Settings.EMPTY);
+        Settings indexSettings = Settings.builder()
+            .put(remoteStoreIndexSettings(0, 1))
+            .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0)
+            .build();
+        createIndex(INDEX_NAME, indexSettings);
+        ensureYellowAndNoInitializingShards(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+
+        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            primaryNodeName(INDEX_NAME)
+        );
+
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+
+        int numDocs = randomIntBetween(5, 10);
+        for (int i = 0; i < numDocs; i++) {
+            keepPinnedTimestampSchedulerUpdated();
+            indexSingleDoc(INDEX_NAME, true);
+        }
+
+        String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings());
+        String shardDataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            DATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1");
+        String shardMetadataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            METADATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath);
+
+        assertBusy(() -> {
+            List<Path> metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList());
+            assertEquals(2, metadataFiles.size());
+
+            verifyTranslogDataFileCount(metadataFiles, translogDataPath);
+        });
+    }
+
+    public void testLiveIndexNoPinnedTimestampsWithExtraGenSettingWithinLimit() throws Exception {
+        prepareCluster(1, 1, Settings.EMPTY);
+        Settings indexSettings = Settings.builder()
+            .put(remoteStoreIndexSettings(0, 1))
+            .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 10)
+            .build();
+        createIndex(INDEX_NAME, indexSettings);
+        ensureYellowAndNoInitializingShards(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+
+        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            primaryNodeName(INDEX_NAME)
+        );
+
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+
+        int numDocs = randomIntBetween(5, 9);
+        for (int i = 0; i < numDocs; i++) {
+            keepPinnedTimestampSchedulerUpdated();
+            indexSingleDoc(INDEX_NAME, true);
+        }
+
+        String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings());
+        String shardDataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            DATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1");
+        String shardMetadataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            METADATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath);
+
+        assertBusy(() -> {
+            List<Path> metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList());
+            assertTrue(metadataFiles.size() >= numDocs + 1);
+
+            verifyTranslogDataFileCount(metadataFiles, translogDataPath);
+        }, 30, TimeUnit.SECONDS);
+    }
+
+    public void testLiveIndexNoPinnedTimestampsWithExtraGenSetting() throws Exception {
+        prepareCluster(1, 1, Settings.EMPTY);
+        Settings indexSettings = Settings.builder()
+            .put(remoteStoreIndexSettings(0, 1))
+            .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 3)
+            .build();
+        createIndex(INDEX_NAME, indexSettings);
+        ensureYellowAndNoInitializingShards(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+
+        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            primaryNodeName(INDEX_NAME)
+        );
+
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+
+        int numDocs = 5;
+        for (int i = 0; i < numDocs; i++) {
+            keepPinnedTimestampSchedulerUpdated();
+            indexSingleDoc(INDEX_NAME, true);
+        }
+
+        String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings());
+        String shardDataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            DATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1");
+        String shardMetadataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            METADATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath);
+
+        assertBusy(() -> {
+            List<Path> metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList());
+            assertEquals(5, metadataFiles.size());
+
+            verifyTranslogDataFileCount(metadataFiles, translogDataPath);
+        });
+    }
+
+    public void testLiveIndexWithPinnedTimestamps() throws Exception {
+        prepareCluster(1, 1, Settings.EMPTY);
+        Settings indexSettings = Settings.builder()
+            .put(remoteStoreIndexSettings(0, 1))
+            .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0)
+            .build();
+        createIndex(INDEX_NAME, indexSettings);
+        ensureYellowAndNoInitializingShards(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+
+        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            primaryNodeName(INDEX_NAME)
+        );
+
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+
+        int numDocs = randomIntBetween(5, 10);
+        for (int i = 0; i < numDocs; i++) {
+            keepPinnedTimestampSchedulerUpdated();
+            indexSingleDoc(INDEX_NAME, true);
+            if (i == 2) {
+                RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueMinutes(1));
+                remoteStorePinnedTimestampService.pinTimestamp(System.currentTimeMillis(), "xyz", noOpActionListener);
+                RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+            }
+        }
+
+        String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings());
+        String shardDataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            DATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1");
+        String shardMetadataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            METADATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath);
+
+        assertBusy(() -> {
+            List<Path> metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList());
+            assertEquals(3, metadataFiles.size());
+
+            verifyTranslogDataFileCount(metadataFiles, translogDataPath);
+        });
+    }
+
+    public void testLiveIndexWithPinnedTimestampsMultiplePrimaryTerms() throws Exception {
+        prepareCluster(1, 2, Settings.EMPTY);
+        Settings indexSettings = Settings.builder()
+            .put(remoteStoreIndexSettings(1, 1))
+            .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 3)
+            .build();
+        createIndex(INDEX_NAME, indexSettings);
+        ensureYellowAndNoInitializingShards(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+
+        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            primaryNodeName(INDEX_NAME)
+        );
+
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+
+        int numDocs = randomIntBetween(5, 10);
+        for (int i = 0; i < numDocs; i++) {
+            keepPinnedTimestampSchedulerUpdated();
+            indexSingleDoc(INDEX_NAME, true);
+            if (i == 2) {
+                RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueMinutes(1));
+                remoteStorePinnedTimestampService.pinTimestamp(System.currentTimeMillis(), "xyz", noOpActionListener);
+                RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+            }
+        }
+
+        ingestDocs();
+
+        internalCluster().restartNode(primaryNodeName(INDEX_NAME));
+        ensureGreen(INDEX_NAME);
+
+        ingestDocs();
+
+        String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings());
+        String shardDataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            DATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1");
+
+        assertBusy(() -> {
+            List<Path> dataFiles = Files.list(translogDataPath).collect(Collectors.toList());
+            assertFalse(dataFiles.isEmpty());
+        });
+    }
+
+    private void ingestDocs() {
+        int numDocs = randomIntBetween(15, 20);
+        for (int i = 0; i < numDocs; i++) {
+            indexSingleDoc(INDEX_NAME, false);
+        }
+
+        assertNoFailures(client().admin().indices().prepareRefresh(INDEX_NAME).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get());
+        flushAndRefresh(INDEX_NAME);
+
+        int numDocsPostFailover = randomIntBetween(15, 20);
+        for (int i = 0; i < numDocsPostFailover; i++) {
+            indexSingleDoc(INDEX_NAME, false);
+        }
+
+        flushAndRefresh(INDEX_NAME);
+        assertNoFailures(client().admin().indices().prepareRefresh(INDEX_NAME).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get());
+    }
+
+    public void testIndexDeletionNoPinnedTimestamps() throws Exception {
+        prepareCluster(1, 1, Settings.EMPTY);
+        Settings indexSettings = Settings.builder()
+            .put(remoteStoreIndexSettings(0, 1))
+            .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0)
+            .build();
+        createIndex(INDEX_NAME, indexSettings);
+        ensureYellowAndNoInitializingShards(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+
+        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            primaryNodeName(INDEX_NAME)
+        );
+
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+
+        int numDocs = randomIntBetween(5, 10);
+        for (int i = 0; i < numDocs; i++) {
+            keepPinnedTimestampSchedulerUpdated();
+            indexSingleDoc(INDEX_NAME, true);
+        }
+
+        String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings());
+        String shardDataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            DATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1");
+        String shardMetadataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            METADATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath);
+
+        assertBusy(() -> {
+            List<Path> metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList());
+            assertEquals(2, metadataFiles.size());
+
+            verifyTranslogDataFileCount(metadataFiles, translogDataPath);
+        });
+
+        keepPinnedTimestampSchedulerUpdated();
+        client().admin().indices().prepareDelete(INDEX_NAME).get();
+
+        assertBusy(() -> {
+            assertEquals(0, Files.list(translogMetadataPath).collect(Collectors.toList()).size());
+            assertEquals(0, Files.list(translogDataPath).collect(Collectors.toList()).size());
+        });
+    }
+
+    public void testIndexDeletionWithPinnedTimestamps() throws Exception {
+        prepareCluster(1, 1, Settings.EMPTY);
+        Settings indexSettings = Settings.builder()
+            .put(remoteStoreIndexSettings(0, 1))
+            .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0)
+            .build();
+        createIndex(INDEX_NAME, indexSettings);
+        ensureYellowAndNoInitializingShards(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+
+        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            primaryNodeName(INDEX_NAME)
+        );
+
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+
+        int numDocs = randomIntBetween(5, 10);
+        for (int i = 0; i < numDocs; i++) {
+            keepPinnedTimestampSchedulerUpdated();
+            indexSingleDoc(INDEX_NAME, true);
+            if (i == 2) {
+                RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueMinutes(1));
+                remoteStorePinnedTimestampService.pinTimestamp(System.currentTimeMillis(), "xyz", noOpActionListener);
+                RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+            }
+        }
+
+        String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings());
+        String shardDataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            DATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1");
+        String shardMetadataPath = getShardLevelBlobPath(
+            client(),
+            INDEX_NAME,
+            BlobPath.cleanPath(),
+            "0",
+            TRANSLOG,
+            METADATA,
+            translogPathFixedPrefix
+        ).buildAsString();
+        Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath);
+
+        assertBusy(() -> {
+            List<Path> metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList());
+            assertEquals(3, metadataFiles.size());
+
+            verifyTranslogDataFileCount(metadataFiles, translogDataPath);
+        }, 30, TimeUnit.SECONDS);
+
+        keepPinnedTimestampSchedulerUpdated();
+        client().admin().indices().prepareDelete(INDEX_NAME).get();
+
+        assertBusy(() -> {
+            List<Path> metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList());
+            assertEquals(1, metadataFiles.size());
+
+            verifyTranslogDataFileCount(metadataFiles, translogDataPath);
+        });
+    }
+
+    private void verifyTranslogDataFileCount(List<Path> metadataFiles, Path translogDataPath) throws IOException {
+        List<String> mdFiles = metadataFiles.stream().map(p -> p.getFileName().toString()).collect(Collectors.toList());
+        Set<Long> generations = new HashSet<>();
+        for (String mdFile : mdFiles) {
+            Tuple<Long, Long> minMaxGen = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(mdFile);
+            generations.addAll(LongStream.rangeClosed(minMaxGen.v1(), minMaxGen.v2()).boxed().collect(Collectors.toList()));
+        }
+        assertEquals(generations.size() * 2, Files.list(translogDataPath).collect(Collectors.toList()).size());
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java
index 2fcda8c2d2f27..3e1127e0ce240 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java
@@ -9,6 +9,8 @@
 package org.opensearch.remotestore;
 
 import org.opensearch.action.LatchedActionListener;
+import org.opensearch.action.admin.cluster.node.stats.NodeStats;
+import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
@@ -17,9 +19,14 @@
 import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 
+import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.REMOTE_STORE;
+
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteStorePinnedTimestampsIT extends RemoteStoreBaseIntegTestCase {
     static final String INDEX_NAME = "remote-store-test-idx-1";
@@ -72,6 +79,11 @@ public void testTimestampPinUnpin() throws Exception {
             Tuple<Long, Set<Long>> pinnedTimestampWithFetchTimestamp_2 = RemoteStorePinnedTimestampService.getPinnedTimestamps();
             long lastFetchTimestamp_2 = pinnedTimestampWithFetchTimestamp_2.v1();
             assertTrue(lastFetchTimestamp_2 != -1);
+            Map<String, List<Long>> pinnedEntities = RemoteStorePinnedTimestampService.getPinnedEntities();
+            assertEquals(3, pinnedEntities.size());
+            assertEquals(Set.of("ss2", "ss3", "ss4"), pinnedEntities.keySet());
+            assertEquals(pinnedEntities.get("ss2").size(), 1);
+            assertEquals(Optional.of(timestamp1).get(), pinnedEntities.get("ss2").get(0));
             assertEquals(Set.of(timestamp1, timestamp2, timestamp3), pinnedTimestampWithFetchTimestamp_2.v2());
         });
 
@@ -99,10 +111,14 @@ public void onFailure(Exception e) {
         // Adding different entity to already pinned timestamp
         remoteStorePinnedTimestampService.pinTimestamp(timestamp3, "ss5", noOpActionListener);
 
-        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+        remoteStorePinnedTimestampService.forceSyncPinnedTimestamps();
 
         assertBusy(() -> {
             Tuple<Long, Set<Long>> pinnedTimestampWithFetchTimestamp_3 = RemoteStorePinnedTimestampService.getPinnedTimestamps();
+            Map<String, List<Long>> pinnedEntities = RemoteStorePinnedTimestampService.getPinnedEntities();
+            assertEquals(3, pinnedEntities.size());
+            assertEquals(pinnedEntities.get("ss5").size(), 1);
+            assertEquals(Optional.of(timestamp3).get(), pinnedEntities.get("ss5").get(0));
             long lastFetchTimestamp_3 = pinnedTimestampWithFetchTimestamp_3.v1();
             assertTrue(lastFetchTimestamp_3 != -1);
             assertEquals(Set.of(timestamp1, timestamp3), pinnedTimestampWithFetchTimestamp_3.v2());
@@ -180,4 +196,41 @@ public void onFailure(Exception e) {
         assertBusy(() -> assertEquals(Set.of(timestamp2, timestamp3), RemoteStorePinnedTimestampService.getPinnedTimestamps().v2()));
         remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3));
     }
+
+    public void testLastSuccessfulFetchOfPinnedTimestampsPresentInNodeStats() throws Exception {
+        logger.info("Starting up cluster manager");
+        logger.info("cluster.remote_store.pinned_timestamps.enabled set to true");
+        logger.info("cluster.remote_store.pinned_timestamps.scheduler_interval set to minimum value of 1minute");
+        Settings pinnedTimestampEnabledSettings = Settings.builder()
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_SCHEDULER_INTERVAL.getKey(), "1m")
+            .build();
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampEnabledSettings);
+        String remoteNodeName = internalCluster().startDataOnlyNodes(1, pinnedTimestampEnabledSettings).get(0);
+        ensureStableCluster(2);
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            remoteNodeName
+        );
+
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+
+        assertBusy(() -> {
+            long lastSuccessfulFetchOfPinnedTimestamps = RemoteStorePinnedTimestampService.getPinnedTimestamps().v1();
+            assertTrue(lastSuccessfulFetchOfPinnedTimestamps > 0L);
+            NodesStatsResponse nodesStatsResponse = internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareNodesStats()
+                .addMetric(REMOTE_STORE.metricName())
+                .execute()
+                .actionGet();
+            for (NodeStats nodeStats : nodesStatsResponse.getNodes()) {
+                long lastRecordedFetch = nodeStats.getRemoteStoreNodeStats().getLastSuccessfulFetchOfPinnedTimestamps();
+                assertTrue(lastRecordedFetch >= lastSuccessfulFetchOfPinnedTimestamps);
+            }
+        });
+
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3));
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java
index c5a55f16cab2b..24f1141ddbede 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java
@@ -10,6 +10,8 @@
 
 import org.opensearch.action.DocWriteResponse;
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.action.admin.indices.recovery.RecoveryResponse;
@@ -19,6 +21,7 @@
 import org.opensearch.client.Requests;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.blobstore.BlobPath;
@@ -64,6 +67,7 @@
 import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
 import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
 import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
+import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -106,6 +110,18 @@ protected Settings.Builder getRepositorySettings(Path location, boolean shallowC
         return settingsBuilder;
     }
 
+    protected Settings.Builder getRepositorySettings(String sourceRepository, boolean readOnly) throws ExecutionException,
+        InterruptedException {
+        GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { sourceRepository });
+        GetRepositoriesResponse res = client().admin().cluster().getRepositories(gr).get();
+        RepositoryMetadata rmd = res.repositories().get(0);
+        return Settings.builder()
+            .put(rmd.settings())
+            .put(BlobStoreRepository.READONLY_SETTING.getKey(), readOnly)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false)
+            .put(SYSTEM_REPOSITORY_SETTING.getKey(), false);
+    }
+
     private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) {
         Settings.Builder settingsBuilder = Settings.builder()
             .put(super.indexSettings())
@@ -802,4 +818,71 @@ public void testInvalidRestoreRequestScenarios() throws Exception {
         );
         assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore"));
     }
+
+    public void testRestoreOperationsUsingDifferentRepos() throws Exception {
+        disableRepoConsistencyCheck("Remote store repo");
+        String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
+        String primary = internalCluster().startDataOnlyNode();
+        String indexName1 = "testindex1";
+        String snapshotRepoName = "test-snapshot-repo";
+        String snapshotName1 = "test-snapshot1";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        // Create repo
+        createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));
+
+        // Create index
+        Client client = client();
+        Settings indexSettings = getIndexSettings(1, 0).build();
+        createIndex(indexName1, indexSettings);
+        ensureGreen(indexName1);
+
+        // Index 5 documents, refresh, index 5 documents
+        final int numDocsInIndex1 = 5;
+        indexDocuments(client, indexName1, 0, numDocsInIndex1);
+        refresh(indexName1);
+        indexDocuments(client, indexName1, numDocsInIndex1, 2 * numDocsInIndex1);
+
+        // Take V2 snapshot
+        logger.info("--> snapshot");
+        SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>());
+        assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(snapshotInfo.successfulShards(), greaterThan(0));
+        assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
+
+        // Create new snapshot, segment and translog repositories
+        String newSnapshotRepo = "backup-snapshot";
+        String newSegmentRepo = "backup-segment";
+        String newTranslogRepo = "backup-translog";
+        createRepository(newSnapshotRepo, "fs", getRepositorySettings(snapshotRepoName, true));
+        createRepository(newSegmentRepo, "fs", getRepositorySettings(BASE_REMOTE_REPO, true));
+        createRepository(newTranslogRepo, "fs", getRepositorySettings(BASE_REMOTE_REPO, true));
+
+        // Delete index
+        assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName1)).get());
+        assertFalse(indexExists(indexName1));
+
+        // Restore using new repos
+        RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(newSnapshotRepo, snapshotName1)
+            .setWaitForCompletion(false)
+            .setIndices(indexName1)
+            .setSourceRemoteStoreRepository(newSegmentRepo)
+            .setSourceRemoteTranslogRepository(newTranslogRepo)
+            .get();
+
+        assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED);
+
+        // Verify restored index's stats
+        ensureYellowAndNoInitializingShards(indexName1);
+        ensureGreen(indexName1);
+        assertDocsPresentInIndex(client(), indexName1, 2 * numDocsInIndex1);
+
+        // indexing some new docs and validating
+        indexDocuments(client, indexName1, 2 * numDocsInIndex1, 3 * numDocsInIndex1);
+        ensureGreen(indexName1);
+        assertDocsPresentInIndex(client, indexName1, 3 * numDocsInIndex1);
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java
index d45b4e3deb798..875f11203281a 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java
@@ -1,3 +1,4 @@
+
 /*
  * SPDX-License-Identifier: Apache-2.0
  *
@@ -12,6 +13,7 @@
 import org.opensearch.common.StreamContext;
 import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
 import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.blobstore.DeleteResult;
 import org.opensearch.common.blobstore.fs.FsBlobContainer;
 import org.opensearch.common.blobstore.fs.FsBlobStore;
 import org.opensearch.common.blobstore.stream.read.ReadContext;
@@ -146,4 +148,14 @@ public boolean remoteIntegrityCheckSupported() {
     private boolean isSegmentFile(String filename) {
         return !filename.endsWith(".tlog") && !filename.endsWith(".ckp");
     }
+
+    @Override
+    public void deleteAsync(ActionListener<DeleteResult> completionListener) {
+        throw new UnsupportedOperationException("deleteAsync");
+    }
+
+    @Override
+    public void deleteBlobsAsyncIgnoringIfNotExists(List<String> blobNames, ActionListener<Void> completionListener) {
+        throw new UnsupportedOperationException("deleteBlobsAsyncIgnoringIfNotExists");
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java
index c6744ae62db60..69e85b13548e0 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java
@@ -34,11 +34,15 @@
 import org.opensearch.action.ActionRunnable;
 import org.opensearch.action.DocWriteResponse;
 import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.opensearch.action.delete.DeleteResponse;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.metadata.RepositoriesMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.rest.RestStatus;
@@ -50,8 +54,10 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import java.nio.file.Path;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
-import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -134,27 +140,32 @@ public void testCloneShallowCopyV2() throws Exception {
         assertTrue(response.isAcknowledged());
         awaitClusterManagerFinishRepoOperations();
 
+        AtomicReference<SnapshotId> cloneSnapshotId = new AtomicReference<>();
         // Validate that snapshot is present in repository data
-        PlainActionFuture<RepositoryData> repositoryDataPlainActionFutureClone = new PlainActionFuture<>();
-        repository.getRepositoryData(repositoryDataPlainActionFutureClone);
-
-        repositoryData = repositoryDataPlainActionFutureClone.get();
-        assertEquals(repositoryData.getSnapshotIds().size(), 2);
-        boolean foundCloneInRepoData = false;
-        SnapshotId cloneSnapshotId = null;
-        for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
-            if (snapshotId.getName().equals("test_clone_snapshot1")) {
-                foundCloneInRepoData = true;
-                cloneSnapshotId = snapshotId;
+        waitUntil(() -> {
+            PlainActionFuture<RepositoryData> repositoryDataPlainActionFutureClone = new PlainActionFuture<>();
+            repository.getRepositoryData(repositoryDataPlainActionFutureClone);
+
+            RepositoryData repositoryData1;
+            try {
+                repositoryData1 = repositoryDataPlainActionFutureClone.get();
+            } catch (InterruptedException | ExecutionException e) {
+                throw new RuntimeException(e);
             }
-        }
-        final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId;
+            for (SnapshotId snapshotId : repositoryData1.getSnapshotIds()) {
+                if (snapshotId.getName().equals("test_clone_snapshot1")) {
+                    cloneSnapshotId.set(snapshotId);
+                    return true;
+                }
+            }
+            return false;
+        }, 90, TimeUnit.SECONDS);
+
+        final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId.get();
         SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get(
             f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal)))
         );
 
-        assertTrue(foundCloneInRepoData);
-
         assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp()));
         for (String index : sourceSnapshotInfo.indices()) {
             assertTrue(cloneSnapshotInfo.indices().contains(index));
@@ -259,19 +270,21 @@ public void testCloneShallowCopyAfterDisablingV2() throws Exception {
         assertThat(sourceSnapshotInfoV1.state(), equalTo(SnapshotState.SUCCESS));
         assertThat(sourceSnapshotInfoV1.successfulShards(), greaterThan(0));
         assertThat(sourceSnapshotInfoV1.successfulShards(), equalTo(sourceSnapshotInfoV1.totalShards()));
-        assertThat(sourceSnapshotInfoV1.getPinnedTimestamp(), equalTo(0L));
+        // assertThat(sourceSnapshotInfoV1.getPinnedTimestamp(), equalTo(0L));
+        AtomicReference<RepositoryData> repositoryDataAtomicReference = new AtomicReference<>();
+        awaitClusterManagerFinishRepoOperations();
 
         // Validate that snapshot is present in repository data
-        PlainActionFuture<RepositoryData> repositoryDataV1PlainActionFuture = new PlainActionFuture<>();
-        BlobStoreRepository repositoryV1 = (BlobStoreRepository) internalCluster().getCurrentClusterManagerNodeInstance(
-            RepositoriesService.class
-        ).repository(snapshotRepoName);
-        repositoryV1.getRepositoryData(repositoryDataV1PlainActionFuture);
-
-        repositoryData = repositoryDataV1PlainActionFuture.get();
+        assertBusy(() -> {
+            Metadata metadata = clusterAdmin().prepareState().get().getState().metadata();
+            RepositoriesMetadata repositoriesMetadata = metadata.custom(RepositoriesMetadata.TYPE);
+            assertEquals(1, repositoriesMetadata.repository(snapshotRepoName).generation());
+            assertEquals(1, repositoriesMetadata.repository(snapshotRepoName).pendingGeneration());
 
-        assertTrue(repositoryData.getSnapshotIds().contains(sourceSnapshotInfoV1.snapshotId()));
-        assertEquals(repositoryData.getSnapshotIds().size(), 2);
+            GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName);
+            GetSnapshotsResponse response = client().admin().cluster().getSnapshots(request).actionGet();
+            assertEquals(2, response.getSnapshots().size());
+        }, 30, TimeUnit.SECONDS);
 
         // clone should get created for v2 snapshot
         AcknowledgedResponse response = client().admin()
@@ -289,31 +302,28 @@ public void testCloneShallowCopyAfterDisablingV2() throws Exception {
         ).repository(snapshotRepoName);
         repositoryCloneV2.getRepositoryData(repositoryDataCloneV2PlainActionFuture);
 
-        repositoryData = repositoryDataCloneV2PlainActionFuture.get();
-
-        assertEquals(repositoryData.getSnapshotIds().size(), 3);
-        boolean foundCloneInRepoData = false;
-        SnapshotId cloneSnapshotId = null;
-        for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
-            if (snapshotId.getName().equals(cloneSnapshotV2)) {
-                foundCloneInRepoData = true;
-                cloneSnapshotId = snapshotId;
-            }
-        }
-        final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId;
-        SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get(
-            f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal)))
-        );
+        // Validate that snapshot is present in repository data
+        assertBusy(() -> {
+            Metadata metadata = clusterAdmin().prepareState().get().getState().metadata();
+            RepositoriesMetadata repositoriesMetadata = metadata.custom(RepositoriesMetadata.TYPE);
+            assertEquals(2, repositoriesMetadata.repository(snapshotRepoName).generation());
+            assertEquals(2, repositoriesMetadata.repository(snapshotRepoName).pendingGeneration());
+            GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName);
+            GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet();
+            assertEquals(3, response2.getSnapshots().size());
+        }, 30, TimeUnit.SECONDS);
 
-        assertTrue(foundCloneInRepoData);
         // pinned timestamp value in clone snapshot v2 matches source snapshot v2
-        assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp()));
-        for (String index : sourceSnapshotInfo.indices()) {
-            assertTrue(cloneSnapshotInfo.indices().contains(index));
-
+        GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName, new String[] { sourceSnapshotV2, cloneSnapshotV2 });
+        GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet();
+
+        SnapshotInfo sourceInfo = response2.getSnapshots().get(0);
+        SnapshotInfo cloneInfo = response2.getSnapshots().get(1);
+        assertEquals(sourceInfo.getPinnedTimestamp(), cloneInfo.getPinnedTimestamp());
+        assertEquals(sourceInfo.totalShards(), cloneInfo.totalShards());
+        for (String index : sourceInfo.indices()) {
+            assertTrue(cloneInfo.indices().contains(index));
         }
-        assertThat(cloneSnapshotInfo.totalShards(), equalTo(sourceSnapshotInfo.totalShards()));
-
     }
 
     public void testRestoreFromClone() throws Exception {
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java
index 15e92f6f7204b..26f8ab5dd8d09 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java
@@ -50,6 +50,7 @@
 import org.opensearch.common.util.concurrent.UncategorizedExecutionException;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.Strings;
+import org.opensearch.core.rest.RestStatus;
 import org.opensearch.discovery.AbstractDisruptionTestCase;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.repositories.RepositoryData;
@@ -135,6 +136,60 @@ public void testLongRunningSnapshotAllowsConcurrentSnapshot() throws Exception {
         assertSuccessful(createSlowFuture);
     }
 
+    public void testSettingsUpdateFailWhenCreateSnapshotInProgress() throws Exception {
+        // Start a cluster with a cluster manager node and a data node
+        internalCluster().startClusterManagerOnlyNode();
+        final String dataNode = internalCluster().startDataOnlyNode();
+        final String repoName = "test-repo";
+        // Create a repository with random settings
+        Settings.Builder settings = randomRepositorySettings();
+        createRepository(repoName, "mock", settings);
+        createIndexWithContent("index");
+        // Start a full snapshot and block it on the data node
+        final ActionFuture<CreateSnapshotResponse> createSlowFuture = startFullSnapshotBlockedOnDataNode(
+            "slow-snapshot",
+            repoName,
+            dataNode
+        );
+        Thread.sleep(1000); // Wait for the snapshot to start
+        assertFalse(createSlowFuture.isDone()); // Ensure the snapshot is still in progress
+        // Attempt to update the repository settings while the snapshot is in progress
+        IllegalStateException ex = assertThrows(IllegalStateException.class, () -> updateRepository(repoName, "mock", settings));
+        // Verify that the update fails with an appropriate exception
+        assertEquals("trying to modify or unregister repository that is currently used", ex.getMessage());
+        unblockNode(repoName, dataNode); // Unblock the snapshot
+        assertSuccessful(createSlowFuture); // Ensure the snapshot completes successfully
+    }
+
+    public void testSettingsUpdateFailWhenDeleteSnapshotInProgress() throws InterruptedException {
+        // Start a cluster with a cluster manager node and a data node
+        String clusterManagerName = internalCluster().startClusterManagerOnlyNode();
+        internalCluster().startDataOnlyNode();
+        final String repoName = "test-repo";
+        // Create a repository with random settings
+        Settings.Builder settings = randomRepositorySettings();
+        createRepository(repoName, "mock", settings);
+        createIndexWithContent("index");
+        final String snapshotName = "snapshot-1";
+        // Create a full snapshot
+        SnapshotInfo snapshotInfo = createFullSnapshot(repoName, snapshotName);
+        assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); // Ensure the snapshot was successful
+        assertEquals(RestStatus.OK, snapshotInfo.status()); // Ensure the snapshot status is OK
+        // Start deleting the snapshot and block it on the cluster manager node
+        ActionFuture<AcknowledgedResponse> future = deleteSnapshotBlockedOnClusterManager(repoName, snapshotName);
+        Thread.sleep(1000); // Wait for the delete operation to start
+        assertFalse(future.isDone()); // Ensure the delete operation is still in progress
+        // Attempt to update the repository settings while the delete operation is in progress
+        IllegalStateException ex = assertThrows(
+            IllegalStateException.class,
+            () -> updateRepository(repoName, "mock", randomRepositorySettings())
+        );
+        // Verify that the update fails with an appropriate exception
+        assertEquals("trying to modify or unregister repository that is currently used", ex.getMessage());
+        unblockNode(repoName, clusterManagerName); // Unblock the delete operation
+        assertAcked(future.actionGet()); // Wait for the delete operation to complete
+    }
+
     public void testDeletesAreBatched() throws Exception {
         internalCluster().startClusterManagerOnlyNode();
         final String dataNode = internalCluster().startDataOnlyNode();
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java
new file mode 100644
index 0000000000000..97d63d0e2380a
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java
@@ -0,0 +1,604 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.snapshots;
+
+import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.opensearch.action.support.PlainActionFuture;
+import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.client.Client;
+import org.opensearch.common.action.ActionFuture;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.remotestore.RemoteSnapshotIT;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.repositories.Repository;
+import org.opensearch.repositories.RepositoryData;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
+import org.opensearch.repositories.fs.FsRepository;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class ConcurrentSnapshotsV2IT extends RemoteSnapshotIT {
+
+    public void testLongRunningSnapshotDontAllowConcurrentSnapshot() throws Exception {
+        final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String repoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(repoName, "mock", settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        blockClusterManagerOnWriteIndexFile(repoName);
+
+        final ActionFuture<CreateSnapshotResponse> snapshotFuture = startFullSnapshot(repoName, "snapshot-queued");
+        awaitNumberOfSnapshotsInProgress(1);
+
+        try {
+            String snapshotName = "snapshot-concurrent";
+            client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName).setWaitForCompletion(true).get();
+            fail();
+        } catch (Exception e) {}
+
+        unblockNode(repoName, clusterManagerName);
+        CreateSnapshotResponse csr = snapshotFuture.actionGet();
+        List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
+        assertEquals(1, snapInfo.size());
+        assertThat(snapInfo, contains(csr.getSnapshotInfo()));
+    }
+
+    public void testCreateSnapshotFailInFinalize() throws Exception {
+        final String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String repoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(repoName, "mock", settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName);
+        final ActionFuture<CreateSnapshotResponse> snapshotFuture = startFullSnapshot(repoName, "snapshot-queued");
+        awaitNumberOfSnapshotsInProgress(1);
+        waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L));
+        unblockNode(repoName, clusterManagerNode);
+        expectThrows(SnapshotException.class, snapshotFuture::actionGet);
+
+        final ActionFuture<CreateSnapshotResponse> snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success");
+        // Second create works out cleanly since the repo
+        CreateSnapshotResponse csr = snapshotFuture2.actionGet();
+
+        List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
+        assertEquals(1, snapInfo.size());
+        assertThat(snapInfo, contains(csr.getSnapshotInfo()));
+    }
+
+    public void testCreateSnapshotV2MasterSwitch() throws Exception {
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String repoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(repoName, "mock", settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        String clusterManagerNode = internalCluster().getClusterManagerName();
+
+        blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName);
+        final ActionFuture<CreateSnapshotResponse> snapshotFuture = startFullSnapshot(repoName, "snapshot-queued");
+        awaitNumberOfSnapshotsInProgress(1);
+        waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L));
+
+        // Fail the cluster manager
+        stopNode(clusterManagerNode);
+
+        ensureGreen();
+
+        final ActionFuture<CreateSnapshotResponse> snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success");
+        // Second create works out cleanly since the repo
+        CreateSnapshotResponse csr = snapshotFuture2.actionGet();
+
+        List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
+        assertEquals(1, snapInfo.size());
+        assertThat(snapInfo, contains(csr.getSnapshotInfo()));
+
+    }
+
+    public void testPinnedTimestampFailSnapshot() throws Exception {
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String repoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(repoName, "mock", settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        // fail segment repo - this is to fail the timestamp pinning
+        setFailRate(BASE_REMOTE_REPO, 100);
+
+        try {
+            String snapshotName = "snapshot-fail";
+            CreateSnapshotResponse createSnapshotResponse = client().admin()
+                .cluster()
+                .prepareCreateSnapshot(repoName, snapshotName)
+                .setWaitForCompletion(true)
+                .get();
+            fail();
+        } catch (Exception e) {}
+
+        setFailRate(BASE_REMOTE_REPO, 0);
+        String snapshotName = "snapshot-success";
+        CreateSnapshotResponse createSnapshotResponse = client().admin()
+            .cluster()
+            .prepareCreateSnapshot(repoName, snapshotName)
+            .setWaitForCompletion(true)
+            .get();
+
+        List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
+        assertEquals(1, snapInfo.size());
+    }
+
+    public void testConcurrentSnapshotV2CreateOperation() throws InterruptedException, ExecutionException {
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String snapshotRepoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(snapshotRepoName, FsRepository.TYPE, settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        int concurrentSnapshots = 5;
+
+        // Prepare threads for concurrent snapshot creation
+        List<Thread> threads = new ArrayList<>();
+
+        for (int i = 0; i < concurrentSnapshots; i++) {
+            int snapshotIndex = i;
+            Thread thread = new Thread(() -> {
+                try {
+                    String snapshotName = "snapshot-concurrent-" + snapshotIndex;
+                    CreateSnapshotResponse createSnapshotResponse2 = client().admin()
+                        .cluster()
+                        .prepareCreateSnapshot(snapshotRepoName, snapshotName)
+                        .setWaitForCompletion(true)
+                        .get();
+                    SnapshotInfo snapshotInfo = createSnapshotResponse2.getSnapshotInfo();
+                    assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
+                    assertThat(snapshotInfo.successfulShards(), greaterThan(0));
+                    assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
+                    assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName));
+                    assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L));
+                } catch (Exception e) {}
+            });
+            threads.add(thread);
+        }
+        // start all threads
+        for (Thread thread : threads) {
+            thread.start();
+        }
+
+        // Wait for all threads to complete
+        for (Thread thread : threads) {
+            thread.join();
+        }
+
+        // Validate that only one snapshot has been created
+        Repository repository = internalCluster().getInstance(RepositoriesService.class).repository(snapshotRepoName);
+        PlainActionFuture<RepositoryData> repositoryDataPlainActionFuture = new PlainActionFuture<>();
+        repository.getRepositoryData(repositoryDataPlainActionFuture);
+
+        RepositoryData repositoryData = repositoryDataPlainActionFuture.get();
+        assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1));
+    }
+
+    public void testLongRunningSnapshotDontAllowConcurrentClone() throws Exception {
+        final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String repoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(repoName, "mock", settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        String sourceSnap = "snapshot-source";
+
+        final CreateSnapshotResponse csr = startFullSnapshot(repoName, sourceSnap).actionGet();
+        blockClusterManagerOnWriteIndexFile(repoName);
+
+        final ActionFuture<AcknowledgedResponse> snapshotFuture = startCloneSnapshot(repoName, sourceSnap, "snapshot-clone");
+        awaitNumberOfSnapshotsInProgress(1);
+
+        final ActionFuture<AcknowledgedResponse> snapshotFuture2 = startCloneSnapshot(repoName, sourceSnap, "snapshot-clone-2");
+        assertThrows(ConcurrentSnapshotExecutionException.class, snapshotFuture2::actionGet);
+
+        unblockNode(repoName, clusterManagerName);
+        assertThrows(SnapshotException.class, snapshotFuture2::actionGet);
+
+        snapshotFuture.get();
+
+        List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
+        assertEquals(2, snapInfo.size());
+    }
+
+    public void testCloneSnapshotFailInFinalize() throws Exception {
+        final String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String repoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(repoName, "mock", settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        String sourceSnap = "snapshot-source";
+        CreateSnapshotResponse sourceResp = startFullSnapshot(repoName, sourceSnap).actionGet();
+
+        blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName);
+        final ActionFuture<AcknowledgedResponse> snapshotFuture = startCloneSnapshot(repoName, sourceSnap, "snapshot-queued");
+        awaitNumberOfSnapshotsInProgress(1);
+        waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L));
+        unblockNode(repoName, clusterManagerNode);
+        assertThrows(SnapshotException.class, snapshotFuture::actionGet);
+
+        final ActionFuture<CreateSnapshotResponse> snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success");
+        // Second create works out cleanly since the repo is cleaned up
+        CreateSnapshotResponse csr = snapshotFuture2.actionGet();
+
+        List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
+        assertEquals(2, snapInfo.size());
+        assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), sourceResp.getSnapshotInfo()));
+    }
+
+    public void testCloneSnapshotV2MasterSwitch() throws Exception {
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String repoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(repoName, "mock", settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        String sourceSnap = "snapshot-source";
+        CreateSnapshotResponse csr = startFullSnapshot(repoName, sourceSnap).actionGet();
+
+        String clusterManagerNode = internalCluster().getClusterManagerName();
+
+        blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName);
+        final ActionFuture<AcknowledgedResponse> snapshotFuture = startCloneSnapshot(repoName, sourceSnap, "snapshot-queued");
+        awaitNumberOfSnapshotsInProgress(1);
+        waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L));
+
+        // Fail the cluster manager
+        stopNode(clusterManagerNode);
+
+        ensureGreen();
+
+        final ActionFuture<CreateSnapshotResponse> snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success");
+        // Second create works out cleanly since the repo
+        CreateSnapshotResponse csr2 = snapshotFuture2.actionGet();
+        List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
+        assertEquals(2, snapInfo.size());
+        assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), csr2.getSnapshotInfo()));
+    }
+
+    public void testDeleteWhileV2CreateOngoing() throws Exception {
+        final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String repoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false);
+        createRepository(repoName, "mock", settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        startFullSnapshot(repoName, "snapshot-v1").actionGet();
+
+        // Creating a v2 repo
+        settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(repoName, "mock", settings);
+
+        blockClusterManagerOnWriteIndexFile(repoName);
+
+        final ActionFuture<CreateSnapshotResponse> snapshotFuture = startFullSnapshot(repoName, "snapshot-v2");
+        awaitNumberOfSnapshotsInProgress(1);
+
+        ActionFuture<AcknowledgedResponse> a = startDeleteSnapshot(repoName, "snapshot-v1");
+
+        unblockNode(repoName, clusterManagerName);
+        CreateSnapshotResponse csr = snapshotFuture.actionGet();
+        assertTrue(csr.getSnapshotInfo().getPinnedTimestamp() != 0);
+        assertTrue(a.actionGet().isAcknowledged());
+        List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
+        assertEquals(1, snapInfo.size());
+        assertThat(snapInfo, contains(csr.getSnapshotInfo()));
+    }
+
+    public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception {
+        final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        internalCluster().startDataOnlyNode(pinnedTimestampSettings());
+        String indexName1 = "testindex1";
+        String indexName2 = "testindex2";
+        String repoName = "test-create-snapshot-repo";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        Settings.Builder settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false);
+        createRepository(repoName, "mock", settings);
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(20, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        Settings indexSettings2 = getIndexSettings(15, 0).build();
+        createIndex(indexName2, indexSettings2);
+
+        final int numDocsInIndex1 = 10;
+        final int numDocsInIndex2 = 20;
+        indexDocuments(client, indexName1, numDocsInIndex1);
+        indexDocuments(client, indexName2, numDocsInIndex2);
+        ensureGreen(indexName1, indexName2);
+
+        startFullSnapshot(repoName, "snapshot-v1").actionGet();
+        startFullSnapshot(repoName, "snapshot-v1-2").actionGet();
+
+        // Creating a v2 repo
+        settings = Settings.builder()
+            .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
+            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+            .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
+            .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
+        createRepository(repoName, "mock", settings);
+
+        blockClusterManagerOnWriteIndexFile(repoName);
+
+        final ActionFuture<CreateSnapshotResponse> snapshotFuture = startFullSnapshot(repoName, "snapshot-v2");
+        awaitNumberOfSnapshotsInProgress(1);
+
+        ActionFuture<AcknowledgedResponse> startDeleteSnapshot = startDeleteSnapshot(repoName, "snapshot-v1");
+        ActionFuture<AcknowledgedResponse> startCloneSnapshot = startCloneSnapshot(repoName, "snapshot-v1-2", "snapshot-v1-2-clone");
+
+        unblockNode(repoName, clusterManagerName);
+        CreateSnapshotResponse csr = snapshotFuture.actionGet();
+        assertTrue(csr.getSnapshotInfo().getPinnedTimestamp() != 0);
+        assertTrue(startDeleteSnapshot.actionGet().isAcknowledged());
+        assertTrue(startCloneSnapshot.actionGet().isAcknowledged());
+        List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
+        assertEquals(3, snapInfo.size());
+    }
+
+    protected ActionFuture<AcknowledgedResponse> startCloneSnapshot(String repoName, String sourceSnapshotName, String snapshotName) {
+        logger.info("--> creating full snapshot [{}] to repo [{}]", snapshotName, repoName);
+        return clusterAdmin().prepareCloneSnapshot(repoName, sourceSnapshotName, snapshotName).setIndices("*").execute();
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java
index 1d7a58384c0be..7b2ad2bccd2b1 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java
@@ -23,19 +23,32 @@
 import org.opensearch.repositories.fs.FsRepository;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
+import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.lessThan;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class DeleteSnapshotV2IT extends AbstractSnapshotIntegTestCase {
 
     private static final String REMOTE_REPO_NAME = "remote-store-repo-name";
 
+    private void keepPinnedTimestampSchedulerUpdated() throws InterruptedException {
+        long currentTime = System.currentTimeMillis();
+        int maxRetry = 10;
+        while (maxRetry > 0 && RemoteStorePinnedTimestampService.getPinnedTimestamps().v1() <= currentTime) {
+            Thread.sleep(1000);
+            maxRetry--;
+        }
+    }
+
     public void testDeleteShallowCopyV2() throws Exception {
         disableRepoConsistencyCheck("Remote store repository is being used in the test");
 
@@ -74,8 +87,8 @@ public void testDeleteShallowCopyV2() throws Exception {
         createIndex(indexName1, getRemoteStoreBackedIndexSettings());
         createIndex(indexName2, getRemoteStoreBackedIndexSettings());
 
-        final int numDocsInIndex1 = 10;
-        final int numDocsInIndex2 = 20;
+        final int numDocsInIndex1 = 1;
+        final int numDocsInIndex2 = 2;
         indexRandomDocs(indexName1, numDocsInIndex1);
         indexRandomDocs(indexName2, numDocsInIndex2);
         ensureGreen(indexName1, indexName2);
@@ -92,7 +105,7 @@ public void testDeleteShallowCopyV2() throws Exception {
         assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName1));
 
         createIndex(indexName3, getRemoteStoreBackedIndexSettings());
-        indexRandomDocs(indexName3, 10);
+        indexRandomDocs(indexName3, 1);
         CreateSnapshotResponse createSnapshotResponse2 = client().admin()
             .cluster()
             .prepareCreateSnapshot(snapshotRepoName, snapshotName2)
@@ -105,109 +118,101 @@ public void testDeleteShallowCopyV2() throws Exception {
         assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2));
 
         assertAcked(client().admin().indices().prepareDelete(indexName1));
-        Thread.sleep(100);
 
-        AcknowledgedResponse deleteResponse = client().admin()
-            .cluster()
-            .prepareDeleteSnapshot(snapshotRepoName, snapshotName2)
-            .setSnapshots(snapshotName2)
-            .get();
+        AcknowledgedResponse deleteResponse = client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, snapshotName2).get();
         assertTrue(deleteResponse.isAcknowledged());
 
         // test delete non-existent snapshot
         assertThrows(
             SnapshotMissingException.class,
-            () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").setSnapshots(snapshotName2).get()
+            () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").get()
         );
-
     }
 
-    public void testDeleteShallowCopyV2MultipleSnapshots() throws Exception {
+    public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exception {
         disableRepoConsistencyCheck("Remote store repository is being used in the test");
         final Path remoteStoreRepoPath = randomRepoPath();
+        Settings settings = remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath);
+        settings = Settings.builder()
+            .put(settings)
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.FIXED.toString())
+            .build();
+        String clusterManagerName = internalCluster().startClusterManagerOnlyNode(settings);
+        internalCluster().startDataOnlyNode(settings);
+        final Client clusterManagerClient = internalCluster().clusterManagerClient();
+        ensureStableCluster(2);
 
-        internalCluster().startClusterManagerOnlyNode(snapshotV2Settings(remoteStoreRepoPath));
-        internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath));
-        internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath));
-
-        String indexName1 = "testindex1";
-        String indexName2 = "testindex2";
-        String indexName3 = "testindex3";
-        String snapshotRepoName = "test-create-snapshot-repo";
-        String snapshotName1 = "test-create-snapshot1";
-        String snapshotName2 = "test-create-snapshot2";
-        Path absolutePath1 = randomRepoPath().toAbsolutePath();
-        logger.info("Snapshot Path [{}]", absolutePath1);
-
-        Client client = client();
-
-        assertAcked(
-            client.admin()
-                .cluster()
-                .preparePutRepository(snapshotRepoName)
-                .setType(FsRepository.TYPE)
-                .setSettings(
-                    Settings.builder()
-                        .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
-                        .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
-                        .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
-                        .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
-                        .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true)
-                )
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            clusterManagerName
         );
 
-        createIndex(indexName1, getRemoteStoreBackedIndexSettings());
+        final String snapshotRepoName = "snapshot-repo-name";
+        final Path snapshotRepoPath = randomRepoPath();
+        createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowV2(snapshotRepoPath));
 
-        createIndex(indexName2, getRemoteStoreBackedIndexSettings());
+        final String remoteStoreEnabledIndexName = "remote-index-1";
+        final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
+        createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
+        indexRandomDocs(remoteStoreEnabledIndexName, 25);
 
-        final int numDocsInIndex1 = 10;
-        final int numDocsInIndex2 = 20;
-        indexRandomDocs(indexName1, numDocsInIndex1);
-        indexRandomDocs(indexName2, numDocsInIndex2);
-        ensureGreen(indexName1, indexName2);
+        String indexUUID = client().admin()
+            .indices()
+            .prepareGetSettings(remoteStoreEnabledIndexName)
+            .get()
+            .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_INDEX_UUID);
 
+        logger.info("--> create two remote index shallow snapshots");
         CreateSnapshotResponse createSnapshotResponse = client().admin()
             .cluster()
-            .prepareCreateSnapshot(snapshotRepoName, snapshotName1)
+            .prepareCreateSnapshot(snapshotRepoName, "snap1")
             .setWaitForCompletion(true)
             .get();
-        SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
-        assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
-        assertThat(snapshotInfo.successfulShards(), greaterThan(0));
-        assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
-        assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName1));
+        SnapshotInfo snapshotInfo1 = createSnapshotResponse.getSnapshotInfo();
 
-        createIndex(indexName3, getRemoteStoreBackedIndexSettings());
-        indexRandomDocs(indexName3, 10);
+        Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID);
+        Path shardPath = Path.of(String.valueOf(indexPath), "0");
 
-        CreateSnapshotResponse createSnapshotResponse2 = client().admin()
-            .cluster()
-            .prepareCreateSnapshot(snapshotRepoName, snapshotName2)
-            .setWaitForCompletion(true)
-            .get();
-        snapshotInfo = createSnapshotResponse2.getSnapshotInfo();
-        assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
-        assertThat(snapshotInfo.successfulShards(), greaterThan(0));
-        assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
-        assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2));
+        // delete remote store index
+        assertAcked(client().admin().indices().prepareDelete(remoteStoreEnabledIndexName));
+
+        logger.info("--> delete snapshot 1");
+
+        Path segmentsPath = Path.of(String.valueOf(shardPath), "segments");
+        Path translogPath = Path.of(String.valueOf(shardPath), "translog");
 
-        AcknowledgedResponse deleteResponse = client().admin()
+        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+
+        keepPinnedTimestampSchedulerUpdated();
+
+        AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin()
             .cluster()
-            .prepareDeleteSnapshot(snapshotRepoName, snapshotName1, snapshotName2)
-            .setSnapshots(snapshotName2)
+            .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo1.snapshotId().getName())
             .get();
-        assertTrue(deleteResponse.isAcknowledged());
+        assertAcked(deleteSnapshotResponse);
 
-        // test delete non-existent snapshot
-        assertThrows(
-            SnapshotMissingException.class,
-            () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").setSnapshots(snapshotName2).get()
-        );
+        // Delete is async. Give time for it
+        assertBusy(() -> {
+            try {
+                assertEquals(0, RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath));
+            } catch (NoSuchFileException e) {
+                fail();
+            }
+        }, 60, TimeUnit.SECONDS);
 
+        assertBusy(() -> {
+            try {
+                assertEquals(0, RemoteStoreBaseIntegTestCase.getFileCount(translogPath));
+            } catch (NoSuchFileException e) {
+                fail();
+            }
+        }, 60, TimeUnit.SECONDS);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15692")
-    public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exception {
+    public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2MultipleSnapshots() throws Exception {
         disableRepoConsistencyCheck("Remote store repository is being used in the test");
         final Path remoteStoreRepoPath = randomRepoPath();
         Settings settings = remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath);
@@ -242,11 +247,11 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio
             .get()
             .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_INDEX_UUID);
 
-        String numShards = client().admin()
-            .indices()
-            .prepareGetSettings(remoteStoreEnabledIndexName)
-            .get()
-            .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_NUMBER_OF_SHARDS);
+        Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID);
+        Path shardPath = Path.of(String.valueOf(indexPath), "0");
+
+        Path segmentsPath = Path.of(String.valueOf(shardPath), "segments", "data");
+        Path translogPath = Path.of(String.valueOf(shardPath), "translog", "data", "1");
 
         logger.info("--> create two remote index shallow snapshots");
         CreateSnapshotResponse createSnapshotResponse = client().admin()
@@ -256,6 +261,11 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio
             .get();
         SnapshotInfo snapshotInfo1 = createSnapshotResponse.getSnapshotInfo();
 
+        List<Path> segmentsPostSnapshot1 = Files.list(segmentsPath).collect(Collectors.toList());
+        List<Path> translogPostSnapshot1 = Files.list(translogPath).collect(Collectors.toList());
+
+        forceMerge(1);
+        refresh(remoteStoreEnabledIndexName);
         indexRandomDocs(remoteStoreEnabledIndexName, 25);
 
         CreateSnapshotResponse createSnapshotResponse2 = client().admin()
@@ -264,70 +274,151 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio
             .setWaitForCompletion(true)
             .get();
         SnapshotInfo snapshotInfo2 = createSnapshotResponse2.getSnapshotInfo();
+
+        List<Path> segmentsPostSnapshot2 = Files.list(segmentsPath).collect(Collectors.toList());
+        List<Path> translogPostSnapshot2 = Files.list(translogPath).collect(Collectors.toList());
+
         assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS));
         assertThat(snapshotInfo2.successfulShards(), greaterThan(0));
         assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards()));
         assertThat(snapshotInfo2.snapshotId().getName(), equalTo("snap2"));
 
+        assertBusy(() -> assertTrue(translogPostSnapshot2.size() > translogPostSnapshot1.size()), 60, TimeUnit.SECONDS);
+        assertBusy(() -> assertTrue(segmentsPostSnapshot2.size() > segmentsPostSnapshot1.size()), 60, TimeUnit.SECONDS);
+
+        keepPinnedTimestampSchedulerUpdated();
+
         // delete remote store index
         assertAcked(client().admin().indices().prepareDelete(remoteStoreEnabledIndexName));
 
-        logger.info("--> delete snapshot 2");
+        logger.info("--> delete snapshot 1");
+        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+        keepPinnedTimestampSchedulerUpdated();
+        // on snapshot deletion, remote store segment files should get cleaned up for deleted index - `remote-index-1`
+        AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin()
+            .cluster()
+            .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo1.snapshotId().getName())
+            .get();
+        assertAcked(deleteSnapshotResponse);
+
+        // Delete is async. Give time for it
+        assertBusy(() -> {
+            List<Path> segmentsPostDeletionOfSnapshot1 = Files.list(segmentsPath).collect(Collectors.toList());
+            assertTrue(segmentsPostDeletionOfSnapshot1.size() < segmentsPostSnapshot2.size());
+        }, 60, TimeUnit.SECONDS);
+        // To uncomment following, we need to handle deletion of generations in translog cleanup flow
+        // List<Path> translogPostDeletionOfSnapshot1 = Files.list(translogPath).collect(Collectors.toList());
+        // Delete is async. Give time for it
+        // assertBusy(() -> assertEquals(translogPostSnapshot2.size() - translogPostSnapshot1.size(),
+        // translogPostDeletionOfSnapshot1.size()), 60, TimeUnit.SECONDS);
+    }
+
+    public void testRemoteStoreCleanupMultiplePrimaryOnSnapshotDeletion() throws Exception {
+        disableRepoConsistencyCheck("Remote store repository is being used in the test");
+        final Path remoteStoreRepoPath = randomRepoPath();
+        Settings settings = remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath);
+        settings = Settings.builder()
+            .put(settings)
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.FIXED.toString())
+            .build();
+        String clusterManagerName = internalCluster().startClusterManagerOnlyNode(settings);
+        internalCluster().startDataOnlyNodes(3, settings);
+        final Client clusterManagerClient = internalCluster().clusterManagerClient();
+        ensureStableCluster(4);
+
+        RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
+            RemoteStorePinnedTimestampService.class,
+            clusterManagerName
+        );
+        remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1));
+        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+
+        final String snapshotRepoName = "snapshot-repo-name";
+        final Path snapshotRepoPath = randomRepoPath();
+        createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowV2(snapshotRepoPath));
+
+        final String remoteStoreEnabledIndexName = "remote-index-1";
+        final Settings remoteStoreEnabledIndexSettings = Settings.builder()
+            .put(getRemoteStoreBackedIndexSettings())
+            .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 2)
+            .build();
+        createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
+        ensureGreen(remoteStoreEnabledIndexName);
+
+        // Create 2 snapshots for primary term 1
+        keepPinnedTimestampSchedulerUpdated();
+        indexRandomDocs(remoteStoreEnabledIndexName, 5);
+        createSnapshot(snapshotRepoName, "snap1");
+        keepPinnedTimestampSchedulerUpdated();
+        indexRandomDocs(remoteStoreEnabledIndexName, 5);
+        createSnapshot(snapshotRepoName, "snap2");
+
+        // Restart current primary to change the primary term
+        internalCluster().restartNode(primaryNodeName(remoteStoreEnabledIndexName));
+        ensureGreen(remoteStoreEnabledIndexName);
+
+        // Create 2 snapshots for primary term 2
+        keepPinnedTimestampSchedulerUpdated();
+        indexRandomDocs(remoteStoreEnabledIndexName, 5);
+        createSnapshot(snapshotRepoName, "snap3");
+        keepPinnedTimestampSchedulerUpdated();
+        indexRandomDocs(remoteStoreEnabledIndexName, 5);
+        createSnapshot(snapshotRepoName, "snap4");
+
+        String indexUUID = client().admin()
+            .indices()
+            .prepareGetSettings(remoteStoreEnabledIndexName)
+            .get()
+            .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_INDEX_UUID);
 
         Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID);
         Path shardPath = Path.of(String.valueOf(indexPath), "0");
-        Path segmentsPath = Path.of(String.valueOf(shardPath), "segments");
-        Path translogPath = Path.of(String.valueOf(shardPath), "translog");
+        Path translogPath = Path.of(String.valueOf(shardPath), "translog", "data", "1");
+
+        // Deleting snap1 will still keep files in primary term 1 due to snap2
+        deleteSnapshot(clusterManagerClient, snapshotRepoName, "snap1");
+        assertTrue(RemoteStoreBaseIntegTestCase.getFileCount(translogPath) > 0);
 
-        // Get total segments remote store directory file count for deleted index and shard 0
-        int segmentFilesCountBeforeDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath);
-        int translogFilesCountBeforeDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(translogPath);
+        // Deleting snap2 will not remove primary term 1 as we need to trigger trimUnreferencedReaders once
+        deleteSnapshot(clusterManagerClient, snapshotRepoName, "snap2");
+        assertTrue(RemoteStoreBaseIntegTestCase.getFileCount(translogPath) > 0);
 
+        // Index a doc to trigger trimUnreferencedReaders
         RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+        keepPinnedTimestampSchedulerUpdated();
+        indexRandomDocs(remoteStoreEnabledIndexName, 5);
 
-        AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin()
+        assertBusy(() -> assertFalse(Files.exists(translogPath)), 30, TimeUnit.SECONDS);
+    }
+
+    private void createSnapshot(String repoName, String snapshotName) {
+        CreateSnapshotResponse createSnapshotResponse = client().admin()
             .cluster()
-            .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo2.snapshotId().getName())
+            .prepareCreateSnapshot(repoName, snapshotName)
+            .setWaitForCompletion(true)
             .get();
-        assertAcked(deleteSnapshotResponse);
-
-        Thread.sleep(5000);
+        SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
 
-        assertBusy(() -> {
-            try {
-                assertThat(RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath), lessThan(segmentFilesCountBeforeDeletingSnapshot1));
-            } catch (Exception e) {}
-        }, 30, TimeUnit.SECONDS);
-        int segmentFilesCountAfterDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath);
+        assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(snapshotInfo.successfulShards(), greaterThan(0));
+        assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
+        assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName));
+    }
 
-        logger.info("--> delete snapshot 1");
-        RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
-        // on snapshot deletion, remote store segment files should get cleaned up for deleted index - `remote-index-1`
-        deleteSnapshotResponse = clusterManagerClient.admin()
+    private void deleteSnapshot(Client clusterManagerClient, String repoName, String snapshotName) {
+        AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin()
             .cluster()
-            .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo1.snapshotId().getName())
+            .prepareDeleteSnapshot(repoName, snapshotName)
             .get();
         assertAcked(deleteSnapshotResponse);
-
-        // Delete is async. Give time for it
-        assertBusy(() -> {
-            try {
-                assertThat(RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath), lessThan(segmentFilesCountAfterDeletingSnapshot1));
-            } catch (Exception e) {}
-        }, 60, TimeUnit.SECONDS);
-
-        assertBusy(() -> {
-            try {
-                assertThat(RemoteStoreBaseIntegTestCase.getFileCount(translogPath), lessThan(translogFilesCountBeforeDeletingSnapshot1));
-            } catch (Exception e) {}
-        }, 60, TimeUnit.SECONDS);
-
     }
 
     private Settings snapshotV2Settings(Path remoteStoreRepoPath) {
         Settings settings = Settings.builder()
             .put(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath))
             .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), false)
             .build();
         return settings;
     }
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java
index c3214022df663..8b6869aa1d81a 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java
@@ -116,7 +116,7 @@ public void testStatusApiConsistency() {
         assertEquals(snapshotStatus.getStats().getTime(), snapshotInfo.endTime() - snapshotInfo.startTime());
     }
 
-    public void testStatusAPICallForShallowCopySnapshot() {
+    public void testStatusAPICallForShallowCopySnapshot() throws Exception {
         disableRepoConsistencyCheck("Remote store repository is being used for the test");
         internalCluster().startClusterManagerOnlyNode();
         internalCluster().startDataOnlyNode();
@@ -136,15 +136,24 @@ public void testStatusAPICallForShallowCopySnapshot() {
         final String snapshot = "snapshot";
         createFullSnapshot(snapshotRepoName, snapshot);
 
-        final SnapshotStatus snapshotStatus = getSnapshotStatus(snapshotRepoName, snapshot);
-        assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS));
+        assertBusy(() -> {
+            final SnapshotStatus snapshotStatus = client().admin()
+                .cluster()
+                .prepareSnapshotStatus(snapshotRepoName)
+                .setSnapshots(snapshot)
+                .execute()
+                .actionGet()
+                .getSnapshots()
+                .get(0);
+            assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS));
 
-        final SnapshotIndexShardStatus snapshotShardState = stateFirstShard(snapshotStatus, indexName);
-        assertThat(snapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE));
-        assertThat(snapshotShardState.getStats().getTotalFileCount(), greaterThan(0));
-        assertThat(snapshotShardState.getStats().getTotalSize(), greaterThan(0L));
-        assertThat(snapshotShardState.getStats().getIncrementalFileCount(), greaterThan(0));
-        assertThat(snapshotShardState.getStats().getIncrementalSize(), greaterThan(0L));
+            final SnapshotIndexShardStatus snapshotShardState = stateFirstShard(snapshotStatus, indexName);
+            assertThat(snapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE));
+            assertThat(snapshotShardState.getStats().getTotalFileCount(), greaterThan(0));
+            assertThat(snapshotShardState.getStats().getTotalSize(), greaterThan(0L));
+            assertThat(snapshotShardState.getStats().getIncrementalFileCount(), greaterThan(0));
+            assertThat(snapshotShardState.getStats().getIncrementalSize(), greaterThan(0L));
+        }, 20, TimeUnit.SECONDS);
     }
 
     public void testStatusAPICallInProgressSnapshot() throws Exception {
@@ -193,7 +202,7 @@ public void testExceptionOnMissingSnapBlob() throws IOException {
         );
     }
 
-    public void testExceptionOnMissingShardLevelSnapBlob() throws IOException {
+    public void testExceptionOnMissingShardLevelSnapBlob() throws Exception {
         disableRepoConsistencyCheck("This test intentionally corrupts the repository");
 
         final Path repoPath = randomRepoPath();
@@ -216,11 +225,12 @@ public void testExceptionOnMissingShardLevelSnapBlob() throws IOException {
             repoPath.resolve(resolvePath(indexId, "0"))
                 .resolve(BlobStoreRepository.SNAPSHOT_PREFIX + snapshotInfo.snapshotId().getUUID() + ".dat")
         );
-
-        expectThrows(
-            SnapshotMissingException.class,
-            () -> client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap").execute().actionGet()
-        );
+        assertBusy(() -> {
+            expectThrows(
+                SnapshotMissingException.class,
+                () -> client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap").execute().actionGet()
+            );
+        }, 20, TimeUnit.SECONDS);
     }
 
     public void testGetSnapshotsWithoutIndices() throws Exception {
diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java
index a1fc61834f69b..0ee3debb4819a 100644
--- a/server/src/main/java/org/opensearch/OpenSearchServerException.java
+++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java
@@ -14,6 +14,7 @@
 import static org.opensearch.Version.V_2_10_0;
 import static org.opensearch.Version.V_2_13_0;
 import static org.opensearch.Version.V_2_17_0;
+import static org.opensearch.Version.V_2_18_0;
 import static org.opensearch.Version.V_2_1_0;
 import static org.opensearch.Version.V_2_4_0;
 import static org.opensearch.Version.V_2_5_0;
@@ -1212,10 +1213,10 @@ public static void registerExceptions() {
         );
         registerExceptionHandle(
             new OpenSearchExceptionHandle(
-                org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException.class,
-                org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException::new,
+                org.opensearch.common.breaker.ResponseLimitBreachedException.class,
+                org.opensearch.common.breaker.ResponseLimitBreachedException::new,
                 175,
-                V_2_17_0
+                V_2_18_0
             )
         );
         registerExceptionHandle(
diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java b/server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java
new file mode 100644
index 0000000000000..d5c521e6c20f3
--- /dev/null
+++ b/server/src/main/java/org/opensearch/accesscontrol/resources/CreatedBy.java
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.accesscontrol.resources;
+
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * This class contains information on the creator of a resource.
+ * Creator can either be a user or a backend_role.
+ *
+ * @opensearch.experimental
+ */
+public class CreatedBy implements ToXContentFragment {
+
+    private String user;
+
+    private String backendRole;
+
+    public CreatedBy(String user, String backendRole) {
+        this.user = user;
+        this.backendRole = backendRole;
+    }
+
+    public String getBackendRole() {
+        return backendRole;
+    }
+
+    public void setBackendRole(String backendRole) {
+        this.backendRole = backendRole;
+    }
+
+    public String getUser() {
+        return user;
+    }
+
+    public void setUser(String user) {
+        this.user = user;
+    }
+
+    @Override
+    public String toString() {
+        return "CreatedBy {" + "user='" + user + '\'' + ", backendRole='" + backendRole + '\'' + '}';
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.startObject().field("user", user).field("backend_role", backendRole).endObject();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/EntityType.java b/server/src/main/java/org/opensearch/accesscontrol/resources/EntityType.java
new file mode 100644
index 0000000000000..435172f4e5097
--- /dev/null
+++ b/server/src/main/java/org/opensearch/accesscontrol/resources/EntityType.java
@@ -0,0 +1,23 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.accesscontrol.resources;
+
+/**
+ * This enum contains the type of entities a resource can be shared with.
+ *
+ * @opensearch.experimental
+ */
+public enum EntityType {
+
+    USERS,
+
+    ROLES,
+
+    BACKEND_ROLES,
+}
diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceAccessScope.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceAccessScope.java
new file mode 100644
index 0000000000000..463936625522c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceAccessScope.java
@@ -0,0 +1,21 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.accesscontrol.resources;
+
+/**
+ * This interface defines the two basic access scopes for resource-access.
+ * Each plugin must implement their own scopes and manage them
+ * These access scopes will then be used to verify the type of access being requested.
+ *
+ * @opensearch.experimental
+ */
+public interface ResourceAccessScope {
+    String READ_ONLY = "read_only";
+    String READ_WRITE = "read_write";
+}
diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java
new file mode 100644
index 0000000000000..41ccd4739d7ec
--- /dev/null
+++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceService.java
@@ -0,0 +1,62 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.accesscontrol.resources;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.OpenSearchException;
+import org.opensearch.plugins.NoOpResourceAccessControlPlugin;
+import org.opensearch.plugins.ResourceAccessControlPlugin;
+import org.opensearch.plugins.ResourcePlugin;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Resource access control for OpenSearch
+ *
+ * @opensearch.experimental
+ * */
+public class ResourceService {
+    private static final Logger log = LogManager.getLogger(ResourceService.class);
+
+    private final ResourceAccessControlPlugin resourceACPlugin;
+    private final List<ResourcePlugin> resourcePlugins;
+
+    public ResourceService(final List<ResourceAccessControlPlugin> resourceACPlugins, List<ResourcePlugin> resourcePlugins) {
+        this.resourcePlugins = resourcePlugins;
+
+        if (resourceACPlugins.isEmpty()) {
+            log.info("Security plugin disabled: Using NoOpResourceAccessControlPlugin");
+            resourceACPlugin = new NoOpResourceAccessControlPlugin();
+        } else if (resourceACPlugins.size() == 1) {
+            log.info("Security plugin enabled: Using OpenSearchSecurityPlugin");
+            resourceACPlugin = resourceACPlugins.get(0);
+        } else {
+            throw new OpenSearchException(
+                "Multiple resource access control plugins are not supported, found: "
+                    + resourceACPlugins.stream().map(Object::getClass).map(Class::getName).collect(Collectors.joining(","))
+            );
+        }
+    }
+
+    /**
+     * Gets the current ResourcePlugin to perform authorization
+     */
+    public ResourceAccessControlPlugin getResourceAccessControlPlugin() {
+        return resourceACPlugin;
+    }
+
+    /**
+     * List active plugins that define resources
+     */
+    List<ResourcePlugin> listResourcePlugins() {
+        return resourcePlugins;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java
new file mode 100644
index 0000000000000..bb5cdfb78a41e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ResourceSharing.java
@@ -0,0 +1,115 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.accesscontrol.resources;
+
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * A document in .resource_sharing index.
+ * Holds information about the resource (obtained from defining plugin's meta-data),
+ * the index which defines the resources, the creator of the resource,
+ * and the information on whom this resource is shared with.
+ *
+ * @opensearch.experimental
+ */
+public class ResourceSharing implements ToXContentFragment {
+
+    private String sourceIdx;
+
+    private String resourceId;
+
+    private CreatedBy createdBy;
+
+    private ShareWith shareWith;
+
+    public ResourceSharing(String sourceIdx, String resourceId, CreatedBy createdBy, ShareWith shareWith) {
+        this.sourceIdx = sourceIdx;
+        this.resourceId = resourceId;
+        this.createdBy = createdBy;
+        this.shareWith = shareWith;
+    }
+
+    public String getSourceIdx() {
+        return sourceIdx;
+    }
+
+    public void setSourceIdx(String sourceIdx) {
+        this.sourceIdx = sourceIdx;
+    }
+
+    public String getResourceId() {
+        return resourceId;
+    }
+
+    public void setResourceId(String resourceId) {
+        this.resourceId = resourceId;
+    }
+
+    public CreatedBy getCreatedBy() {
+        return createdBy;
+    }
+
+    public void setCreatedBy(CreatedBy createdBy) {
+        this.createdBy = createdBy;
+    }
+
+    public ShareWith getShareWith() {
+        return shareWith;
+    }
+
+    public void setShareWith(ShareWith shareWith) {
+        this.shareWith = shareWith;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ResourceSharing resourceSharing = (ResourceSharing) o;
+        return Objects.equals(getSourceIdx(), resourceSharing.getSourceIdx())
+            && Objects.equals(getResourceId(), resourceSharing.getResourceId())
+            && Objects.equals(getCreatedBy(), resourceSharing.getCreatedBy())
+            && Objects.equals(getShareWith(), resourceSharing.getShareWith());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getSourceIdx(), getResourceId(), getCreatedBy(), getShareWith());
+    }
+
+    @Override
+    public String toString() {
+        return "Resource {"
+            + "sourceIdx='"
+            + sourceIdx
+            + '\''
+            + ", resourceId='"
+            + resourceId
+            + '\''
+            + ", createdBy="
+            + createdBy
+            + ", sharedWith="
+            + shareWith
+            + '}';
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.startObject()
+            .field("source_idx", sourceIdx)
+            .field("resource_id", resourceId)
+            .field("created_by", createdBy)
+            .field("share_with", shareWith)
+            .endObject();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java
new file mode 100644
index 0000000000000..2018cdfdfd96d
--- /dev/null
+++ b/server/src/main/java/org/opensearch/accesscontrol/resources/ShareWith.java
@@ -0,0 +1,63 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.accesscontrol.resources;
+
+import org.opensearch.core.common.io.stream.NamedWriteable;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * This class contains information about whom a resource is shared with and at what scope.
+ * Here is a sample of what this would look like:
+ * "share_with": {
+ *       "read_only": {
+ *          "users": [],
+ *          "roles": [],
+ *          "backend_roles": []
+ *       },
+ *       "read_write": {
+ *          "users": [],
+ *          "roles": [],
+ *          "backend_roles": []
+ *       }
+ *    }
+ *
+ * @opensearch.experimental
+ */
+public class ShareWith implements ToXContentFragment, NamedWriteable {
+
+    private final List<SharedWithScope> sharedWithScopes;
+
+    public ShareWith(List<SharedWithScope> sharedWithScopes) {
+        this.sharedWithScopes = sharedWithScopes;
+    }
+
+    public List<SharedWithScope> getSharedWithScopes() {
+        return sharedWithScopes;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.startObject("share_with").value(sharedWithScopes).endObject();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return "share_with";
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeList(sharedWithScopes);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java b/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java
new file mode 100644
index 0000000000000..1f424177fcd4e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/accesscontrol/resources/SharedWithScope.java
@@ -0,0 +1,128 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.accesscontrol.resources;
+
+import org.opensearch.core.common.io.stream.NamedWriteable;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Defines the scope and who this scope is shared with
+ *
+ * @opensearch.experimental
+ */
+public class SharedWithScope implements ToXContentFragment, NamedWriteable {
+
+    private final String scope;
+
+    private final SharedWithPerScope sharedWithPerScope;
+
+    public SharedWithScope(String scope, SharedWithPerScope sharedWithPerScope) {
+        this.scope = scope;
+        this.sharedWithPerScope = sharedWithPerScope;
+    }
+
+    public String getScope() {
+        return scope;
+    }
+
+    public SharedWithPerScope getSharedWithPerScope() {
+        return sharedWithPerScope;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.startObject().field(scope, sharedWithPerScope).endObject();
+    }
+
+    @Override
+    public String toString() {
+        return "SharedWithScope {" + scope + ": " + sharedWithPerScope + '}';
+    }
+
+    @Override
+    public String getWriteableName() {
+        return "shared_with_scope";
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(scope);
+        out.writeNamedWriteable(sharedWithPerScope);
+    }
+
+    /**
+     * This class defines who a resource is shared_with for a particular scope
+     *
+     * @opensearch.experimental
+     */
+    public static class SharedWithPerScope implements ToXContentFragment, NamedWriteable {
+        private List<String> users;
+
+        private List<String> roles;
+
+        private List<String> backendRoles;
+
+        public SharedWithPerScope(List<String> users, List<String> roles, List<String> backendRoles) {
+            this.users = users;
+            this.roles = roles;
+            this.backendRoles = backendRoles;
+        }
+
+        public List<String> getUsers() {
+            return users;
+        }
+
+        public void setUsers(List<String> users) {
+            this.users = users;
+        }
+
+        public List<String> getRoles() {
+            return roles;
+        }
+
+        public void setRoles(List<String> roles) {
+            this.roles = roles;
+        }
+
+        public List<String> getBackendRoles() {
+            return backendRoles;
+        }
+
+        public void setBackendRoles(List<String> backendRoles) {
+            this.backendRoles = backendRoles;
+        }
+
+        @Override
+        public String toString() {
+            return "ShareWith {" + "users=" + users + ", roles=" + roles + ", backendRoles=" + backendRoles + '}';
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.startObject().field("users", users).field("roles", roles).field("backend_roles", backendRoles).endObject();
+        }
+
+        @Override
+        public String getWriteableName() {
+            return "shared_with_per_scope";
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeStringArray(users.toArray(new String[0]));
+            out.writeStringArray(roles.toArray(new String[0]));
+            out.writeStringArray(backendRoles.toArray(new String[0]));
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java b/server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java
new file mode 100644
index 0000000000000..b8ad5237a60ca
--- /dev/null
+++ b/server/src/main/java/org/opensearch/accesscontrol/resources/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * This package defines all classes required for Resource Sharing and Access Control
+ */
+package org.opensearch.accesscontrol.resources;
diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java
index fbf90b97d1e8f..98bcd6ba9c3f8 100644
--- a/server/src/main/java/org/opensearch/action/ActionModule.java
+++ b/server/src/main/java/org/opensearch/action/ActionModule.java
@@ -300,6 +300,7 @@
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.NamedRegistry;
 import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.breaker.ResponseLimitSettings;
 import org.opensearch.common.inject.AbstractModule;
 import org.opensearch.common.inject.TypeLiteral;
 import org.opensearch.common.inject.multibindings.MapBinder;
@@ -461,6 +462,9 @@
 import org.opensearch.rest.action.ingest.RestGetPipelineAction;
 import org.opensearch.rest.action.ingest.RestPutPipelineAction;
 import org.opensearch.rest.action.ingest.RestSimulatePipelineAction;
+import org.opensearch.rest.action.list.AbstractListAction;
+import org.opensearch.rest.action.list.RestIndicesListAction;
+import org.opensearch.rest.action.list.RestListAction;
 import org.opensearch.rest.action.search.RestClearScrollAction;
 import org.opensearch.rest.action.search.RestCountAction;
 import org.opensearch.rest.action.search.RestCreatePitAction;
@@ -528,6 +532,7 @@ public class ActionModule extends AbstractModule {
     private final RequestValidators<IndicesAliasesRequest> indicesAliasesRequestRequestValidators;
     private final ThreadPool threadPool;
     private final ExtensionsManager extensionsManager;
+    private final ResponseLimitSettings responseLimitSettings;
 
     public ActionModule(
         Settings settings,
@@ -579,7 +584,8 @@ public ActionModule(
             actionPlugins.stream().flatMap(p -> p.indicesAliasesRequestValidators().stream()).collect(Collectors.toList())
         );
 
-        restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService, identityService);
+        restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService);
+        responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings);
     }
 
     public Map<String, ActionHandler<?, ?>> getActions() {
@@ -802,9 +808,14 @@ private ActionFilters setupActionFilters(List<ActionPlugin> actionPlugins) {
 
     public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         List<AbstractCatAction> catActions = new ArrayList<>();
+        List<AbstractListAction> listActions = new ArrayList<>();
         Consumer<RestHandler> registerHandler = handler -> {
             if (handler instanceof AbstractCatAction) {
-                catActions.add((AbstractCatAction) handler);
+                if (handler instanceof AbstractListAction && ((AbstractListAction) handler).isActionPaginated()) {
+                    listActions.add((AbstractListAction) handler);
+                } else {
+                    catActions.add((AbstractCatAction) handler);
+                }
             }
             restController.registerHandler(handler);
         };
@@ -960,8 +971,8 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestClusterManagerAction());
         registerHandler.accept(new RestNodesAction());
         registerHandler.accept(new RestTasksAction(nodesInCluster));
-        registerHandler.accept(new RestIndicesAction());
-        registerHandler.accept(new RestSegmentsAction());
+        registerHandler.accept(new RestIndicesAction(responseLimitSettings));
+        registerHandler.accept(new RestSegmentsAction(responseLimitSettings));
         // Fully qualified to prevent interference with rest.action.count.RestCountAction
         registerHandler.accept(new org.opensearch.rest.action.cat.RestCountAction());
         // Fully qualified to prevent interference with rest.action.indices.RestRecoveryAction
@@ -980,6 +991,9 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         }
         registerHandler.accept(new RestTemplatesAction());
 
+        // LIST API
+        registerHandler.accept(new RestIndicesListAction(responseLimitSettings));
+
         // Point in time API
         registerHandler.accept(new RestCreatePitAction());
         registerHandler.accept(new RestDeletePitAction());
@@ -1011,6 +1025,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
             }
         }
         registerHandler.accept(new RestCatAction(catActions));
+        registerHandler.accept(new RestListAction(listActions));
         registerHandler.accept(new RestDecommissionAction());
         registerHandler.accept(new RestGetDecommissionStateAction());
         registerHandler.accept(new RestRemoteStoreStatsAction());
@@ -1048,6 +1063,8 @@ protected void configure() {
 
         // register dynamic ActionType -> transportAction Map used by NodeClient
         bind(DynamicActionRegistry.class).toInstance(dynamicActionRegistry);
+
+        bind(ResponseLimitSettings.class).toInstance(responseLimitSettings);
     }
 
     public ActionFilters getActionFilters() {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java
index aab7ea54f87c2..d694721feda38 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java
@@ -60,7 +60,7 @@ public class NodesReloadSecureSettingsRequest extends BaseNodesRequest<NodesRelo
     private SecureString secureSettingsPassword;
 
     public NodesReloadSecureSettingsRequest() {
-        super((String[]) null);
+        super(true, (String[]) null);
     }
 
     public NodesReloadSecureSettingsRequest(StreamInput in) throws IOException {
@@ -84,7 +84,7 @@ public NodesReloadSecureSettingsRequest(StreamInput in) throws IOException {
      * nodes.
      */
     public NodesReloadSecureSettingsRequest(String... nodesIds) {
-        super(nodesIds);
+        super(true, nodesIds);
     }
 
     @Nullable
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
index 0917a0baff1ab..eb79e3403a25c 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
@@ -59,6 +59,7 @@
 import org.opensearch.monitor.process.ProcessStats;
 import org.opensearch.node.AdaptiveSelectionStats;
 import org.opensearch.node.NodesResourceUsageStats;
+import org.opensearch.node.remotestore.RemoteStoreNodeStats;
 import org.opensearch.ratelimitting.admissioncontrol.stats.AdmissionControlStats;
 import org.opensearch.repositories.RepositoriesStats;
 import org.opensearch.script.ScriptCacheStats;
@@ -162,6 +163,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
     @Nullable
     private NodeCacheStats nodeCacheStats;
 
+    @Nullable
+    private RemoteStoreNodeStats remoteStoreNodeStats;
+
     public NodeStats(StreamInput in) throws IOException {
         super(in);
         timestamp = in.readVLong();
@@ -243,6 +247,11 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             nodeCacheStats = null;
         }
+        if (in.getVersion().onOrAfter(Version.V_2_18_0)) {
+            remoteStoreNodeStats = in.readOptionalWriteable(RemoteStoreNodeStats::new);
+        } else {
+            remoteStoreNodeStats = null;
+        }
     }
 
     public NodeStats(
@@ -274,7 +283,8 @@ public NodeStats(
         @Nullable SegmentReplicationRejectionStats segmentReplicationRejectionStats,
         @Nullable RepositoriesStats repositoriesStats,
         @Nullable AdmissionControlStats admissionControlStats,
-        @Nullable NodeCacheStats nodeCacheStats
+        @Nullable NodeCacheStats nodeCacheStats,
+        @Nullable RemoteStoreNodeStats remoteStoreNodeStats
     ) {
         super(node);
         this.timestamp = timestamp;
@@ -305,6 +315,7 @@ public NodeStats(
         this.repositoriesStats = repositoriesStats;
         this.admissionControlStats = admissionControlStats;
         this.nodeCacheStats = nodeCacheStats;
+        this.remoteStoreNodeStats = remoteStoreNodeStats;
     }
 
     public long getTimestamp() {
@@ -467,6 +478,11 @@ public NodeCacheStats getNodeCacheStats() {
         return nodeCacheStats;
     }
 
+    @Nullable
+    public RemoteStoreNodeStats getRemoteStoreNodeStats() {
+        return remoteStoreNodeStats;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -525,6 +541,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_14_0)) {
             out.writeOptionalWriteable(nodeCacheStats);
         }
+        if (out.getVersion().onOrAfter(Version.V_2_18_0)) {
+            out.writeOptionalWriteable(remoteStoreNodeStats);
+        }
     }
 
     @Override
@@ -631,6 +650,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (getNodeCacheStats() != null) {
             getNodeCacheStats().toXContent(builder, params);
         }
+        if (getRemoteStoreNodeStats() != null) {
+            getRemoteStoreNodeStats().toXContent(builder, params);
+        }
         return builder;
     }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
index f1f9f93afdad2..a5b00ed82d3cb 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -220,7 +220,8 @@ public enum Metric {
         SEGMENT_REPLICATION_BACKPRESSURE("segment_replication_backpressure"),
         REPOSITORIES("repositories"),
         ADMISSION_CONTROL("admission_control"),
-        CACHE_STATS("caches");
+        CACHE_STATS("caches"),
+        REMOTE_STORE("remote_store");
 
         private String metricName;
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index 2c808adc97c7a..a98d245af872b 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -129,7 +129,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
             NodesStatsRequest.Metric.SEGMENT_REPLICATION_BACKPRESSURE.containedIn(metrics),
             NodesStatsRequest.Metric.REPOSITORIES.containedIn(metrics),
             NodesStatsRequest.Metric.ADMISSION_CONTROL.containedIn(metrics),
-            NodesStatsRequest.Metric.CACHE_STATS.containedIn(metrics)
+            NodesStatsRequest.Metric.CACHE_STATS.containedIn(metrics),
+            NodesStatsRequest.Metric.REMOTE_STORE.containedIn(metrics)
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java
index 49299777db8ae..76aa25b9a96b5 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java
@@ -27,11 +27,13 @@ public class CatShardsRequest extends ClusterManagerNodeReadRequest<CatShardsReq
 
     private String[] indices;
     private TimeValue cancelAfterTimeInterval;
+    private boolean requestLimitCheckSupported;
 
     public CatShardsRequest() {}
 
     public CatShardsRequest(StreamInput in) throws IOException {
         super(in);
+        this.requestLimitCheckSupported = false;
     }
 
     @Override
@@ -55,6 +57,14 @@ public TimeValue getCancelAfterTimeInterval() {
         return this.cancelAfterTimeInterval;
     }
 
+    public void setRequestLimitCheckSupported(final boolean requestLimitCheckSupported) {
+        this.requestLimitCheckSupported = requestLimitCheckSupported;
+    }
+
+    public boolean isRequestLimitCheckSupported() {
+        return this.requestLimitCheckSupported;
+    }
+
     @Override
     public ClusterAdminTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
         return new ClusterAdminTask(id, type, action, parentTaskId, headers, this.cancelAfterTimeInterval);
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java
index 224d3cbc5f10a..6b073a16f7d28 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java
@@ -16,6 +16,8 @@
 import org.opensearch.action.support.HandledTransportAction;
 import org.opensearch.action.support.TimeoutTaskCancellationUtility;
 import org.opensearch.client.node.NodeClient;
+import org.opensearch.common.breaker.ResponseLimitBreachedException;
+import org.opensearch.common.breaker.ResponseLimitSettings;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.action.NotifyOnceListener;
@@ -23,6 +25,10 @@
 import org.opensearch.tasks.Task;
 import org.opensearch.transport.TransportService;
 
+import java.util.Objects;
+
+import static org.opensearch.common.breaker.ResponseLimitSettings.LimitEntity.SHARDS;
+
 /**
  * Perform cat shards action
  *
@@ -31,11 +37,18 @@
 public class TransportCatShardsAction extends HandledTransportAction<CatShardsRequest, CatShardsResponse> {
 
     private final NodeClient client;
+    private final ResponseLimitSettings responseLimitSettings;
 
     @Inject
-    public TransportCatShardsAction(NodeClient client, TransportService transportService, ActionFilters actionFilters) {
+    public TransportCatShardsAction(
+        NodeClient client,
+        TransportService transportService,
+        ActionFilters actionFilters,
+        ResponseLimitSettings responseLimitSettings
+    ) {
         super(CatShardsAction.NAME, transportService, actionFilters, CatShardsRequest::new);
         this.client = client;
+        this.responseLimitSettings = responseLimitSettings;
     }
 
     @Override
@@ -73,6 +86,7 @@ protected void innerOnFailure(Exception e) {
             client.admin().cluster().state(clusterStateRequest, new ActionListener<ClusterStateResponse>() {
                 @Override
                 public void onResponse(ClusterStateResponse clusterStateResponse) {
+                    validateRequestLimit(shardsRequest, clusterStateResponse, cancellableListener);
                     catShardsResponse.setClusterStateResponse(clusterStateResponse);
                     IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
                     indicesStatsRequest.setShouldCancelOnTimeout(true);
@@ -107,4 +121,19 @@ public void onFailure(Exception e) {
         }
 
     }
+
+    private void validateRequestLimit(
+        final CatShardsRequest shardsRequest,
+        final ClusterStateResponse clusterStateResponse,
+        final ActionListener<CatShardsResponse> listener
+    ) {
+        if (shardsRequest.isRequestLimitCheckSupported()
+            && Objects.nonNull(clusterStateResponse)
+            && Objects.nonNull(clusterStateResponse.getState())) {
+            int limit = responseLimitSettings.getCatShardsResponseLimit();
+            if (ResponseLimitSettings.isResponseLimitBreached(clusterStateResponse.getState().getRoutingTable(), SHARDS, limit)) {
+                listener.onFailure(new ResponseLimitBreachedException("Too many shards requested.", limit, SHARDS));
+            }
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index 469a31407dc5d..f3110cc8f20a5 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -652,6 +652,12 @@ public RestoreSnapshotRequest source(Map<String, Object> source) {
                 } else {
                     throw new IllegalArgumentException("malformed source_remote_store_repository");
                 }
+            } else if (name.equals("source_remote_translog_repository")) {
+                if (entry.getValue() instanceof String) {
+                    setSourceRemoteTranslogRepository((String) entry.getValue());
+                } else {
+                    throw new IllegalArgumentException("malformed source_remote_translog_repository");
+                }
             } else {
                 if (IndicesOptions.isIndicesOptions(name) == false) {
                     throw new IllegalArgumentException("Unknown parameter " + name);
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
index 39eaadf3c8de6..53c9557a621b7 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
@@ -266,4 +266,12 @@ public RestoreSnapshotRequestBuilder setSourceRemoteStoreRepository(String repos
         request.setSourceRemoteStoreRepository(repositoryName);
         return this;
     }
+
+    /**
+     * Sets the source remote translog repository name
+     */
+    public RestoreSnapshotRequestBuilder setSourceRemoteTranslogRepository(String repositoryName) {
+        request.setSourceRemoteTranslogRepository(repositoryName);
+        return this;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index a49ca2035783c..c4b3524cf6da5 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -174,6 +174,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             false,
             false,
             false,
+            false,
             false
         );
         List<ShardStats> shardsStats = new ArrayList<>();
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
index a8b97d0f344ae..1a3c657f5b1b8 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.FailedNodeException;
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeReadAction;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.block.ClusterBlockException;
 import org.opensearch.cluster.block.ClusterBlockLevel;
@@ -88,6 +89,7 @@ public class TransportIndicesShardStoresAction extends TransportClusterManagerNo
     private static final Logger logger = LogManager.getLogger(TransportIndicesShardStoresAction.class);
 
     private final TransportNodesListGatewayStartedShards listShardStoresInfo;
+    private final ClusterManagerMetrics clusterManagerMetrics;
 
     @Inject
     public TransportIndicesShardStoresAction(
@@ -96,7 +98,8 @@ public TransportIndicesShardStoresAction(
         ThreadPool threadPool,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
-        TransportNodesListGatewayStartedShards listShardStoresInfo
+        TransportNodesListGatewayStartedShards listShardStoresInfo,
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
         super(
             IndicesShardStoresAction.NAME,
@@ -109,6 +112,7 @@ public TransportIndicesShardStoresAction(
             true
         );
         this.listShardStoresInfo = listShardStoresInfo;
+        this.clusterManagerMetrics = clusterManagerMetrics;
     }
 
     @Override
@@ -154,7 +158,7 @@ protected void clusterManagerOperation(
         // we could fetch all shard store info from every node once (nNodes requests)
         // we have to implement a TransportNodesAction instead of using TransportNodesListGatewayStartedShards
         // for fetching shard stores info, that operates on a list of shards instead of a single shard
-        new AsyncShardStoresInfoFetches(state.nodes(), routingNodes, shardsToFetch, listener).start();
+        new AsyncShardStoresInfoFetches(state.nodes(), routingNodes, shardsToFetch, listener, clusterManagerMetrics).start();
     }
 
     @Override
@@ -175,12 +179,14 @@ private class AsyncShardStoresInfoFetches {
         private final ActionListener<IndicesShardStoresResponse> listener;
         private CountDown expectedOps;
         private final Queue<InternalAsyncFetch.Response> fetchResponses;
+        private final ClusterManagerMetrics clusterManagerMetrics;
 
         AsyncShardStoresInfoFetches(
             DiscoveryNodes nodes,
             RoutingNodes routingNodes,
             Set<Tuple<ShardId, String>> shards,
-            ActionListener<IndicesShardStoresResponse> listener
+            ActionListener<IndicesShardStoresResponse> listener,
+            ClusterManagerMetrics clusterManagerMetrics
         ) {
             this.nodes = nodes;
             this.routingNodes = routingNodes;
@@ -188,6 +194,7 @@ private class AsyncShardStoresInfoFetches {
             this.listener = listener;
             this.fetchResponses = new ConcurrentLinkedQueue<>();
             this.expectedOps = new CountDown(shards.size());
+            this.clusterManagerMetrics = clusterManagerMetrics;
         }
 
         void start() {
@@ -195,7 +202,14 @@ void start() {
                 listener.onResponse(new IndicesShardStoresResponse());
             } else {
                 for (Tuple<ShardId, String> shard : shards) {
-                    InternalAsyncFetch fetch = new InternalAsyncFetch(logger, "shard_stores", shard.v1(), shard.v2(), listShardStoresInfo);
+                    InternalAsyncFetch fetch = new InternalAsyncFetch(
+                        logger,
+                        "shard_stores",
+                        shard.v1(),
+                        shard.v2(),
+                        listShardStoresInfo,
+                        clusterManagerMetrics
+                    );
                     fetch.fetchData(nodes, Collections.emptyMap());
                 }
             }
@@ -213,9 +227,10 @@ private class InternalAsyncFetch extends AsyncShardFetch<NodeGatewayStartedShard
                 String type,
                 ShardId shardId,
                 String customDataPath,
-                TransportNodesListGatewayStartedShards action
+                TransportNodesListGatewayStartedShards action,
+                ClusterManagerMetrics clusterManagerMetrics
             ) {
-                super(logger, type, shardId, customDataPath, action);
+                super(logger, type, shardId, customDataPath, action, clusterManagerMetrics);
             }
 
             @Override
diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
index f0fc05c595d6f..836083f91b995 100644
--- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
@@ -754,6 +754,9 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At
             }
             searchRequestContext.setTotalHits(internalSearchResponse.hits().getTotalHits());
             searchRequestContext.setShardStats(results.getNumShards(), successfulOps.get(), skippedOps.get(), failures.length);
+            searchRequestContext.setSuccessfulSearchShardIndices(
+                results.getSuccessfulResults().map(result -> result.getSearchShardTarget().getIndex()).collect(Collectors.toSet())
+            );
             onPhaseEnd(searchRequestContext);
             onRequestEnd(searchRequestContext);
             listener.onResponse(buildSearchResponse(internalSearchResponse, failures, scrollId, searchContextId));
diff --git a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java
index 5b887b48f696e..f16d7d1e7d6a3 100644
--- a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java
@@ -310,6 +310,10 @@ public static void readMultiLineFormat(
             ) {
                 consumer.accept(searchRequest, parser);
             }
+
+            if (searchRequest.source() != null && searchRequest.source().pipeline() != null) {
+                searchRequest.pipeline(searchRequest.source().pipeline());
+            }
             // move pointers
             from = nextMarker + 1;
         }
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java
index 111d9c64550b3..376cf71448d5c 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java
@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.function.Supplier;
 
@@ -36,6 +37,7 @@ public class SearchRequestContext {
     private final Map<String, Long> phaseTookMap;
     private TotalHits totalHits;
     private final EnumMap<ShardStatsFieldNames, Integer> shardStats;
+    private Set<String> successfulSearchShardIndices;
 
     private final SearchRequest searchRequest;
     private final LinkedBlockingQueue<TaskResourceInfo> phaseResourceUsage;
@@ -141,6 +143,18 @@ public List<TaskResourceInfo> getPhaseResourceUsage() {
     public SearchRequest getRequest() {
         return searchRequest;
     }
+
+    void setSuccessfulSearchShardIndices(Set<String> successfulSearchShardIndices) {
+        this.successfulSearchShardIndices = successfulSearchShardIndices;
+    }
+
+    /**
+     * @return A {@link List} of {@link String} representing the names of the indices that were
+     * successfully queried at the shard level.
+     */
+    public Set<String> getSuccessfulSearchShardIndices() {
+        return successfulSearchShardIndices;
+    }
 }
 
 enum ShardStatsFieldNames {
diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java
index cf8e6085d4b32..f3fd756c6db4b 100644
--- a/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java
+++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java
@@ -41,7 +41,7 @@ public GetTermVersionResponse(ClusterStateTermVersion clusterStateTermVersion, b
     public GetTermVersionResponse(StreamInput in) throws IOException {
         super(in);
         this.clusterStateTermVersion = new ClusterStateTermVersion(in);
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_18_0)) {
             this.isStatePresentInRemote = in.readOptionalBoolean();
         } else {
             this.isStatePresentInRemote = false;
@@ -51,7 +51,7 @@ public GetTermVersionResponse(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         clusterStateTermVersion.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_18_0)) {
             out.writeOptionalBoolean(isStatePresentInRemote);
         }
     }
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java b/server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java
index a98349a4af5cd..ba63b3fd2d6d0 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java
@@ -34,6 +34,8 @@ public final class ClusterManagerMetrics {
 
     public final Counter leaderCheckFailureCounter;
     public final Counter followerChecksFailureCounter;
+    public final Counter asyncFetchFailureCounter;
+    public final Counter asyncFetchSuccessCounter;
 
     public ClusterManagerMetrics(MetricsRegistry metricsRegistry) {
         clusterStateAppliersHistogram = metricsRegistry.createHistogram(
@@ -71,6 +73,17 @@ public ClusterManagerMetrics(MetricsRegistry metricsRegistry) {
             "Counter for number of failed leader checks",
             COUNTER_METRICS_UNIT
         );
+        asyncFetchFailureCounter = metricsRegistry.createCounter(
+            "async.fetch.failure.count",
+            "Counter for number of failed async fetches",
+            COUNTER_METRICS_UNIT
+        );
+        asyncFetchSuccessCounter = metricsRegistry.createCounter(
+            "async.fetch.success.count",
+            "Counter for number of successful async fetches",
+            COUNTER_METRICS_UNIT
+        );
+
     }
 
     public void recordLatency(Histogram histogram, Double value) {
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java
index 3924e8e0589a7..66b1740c8fa96 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java
@@ -142,6 +142,7 @@ public class ClusterModule extends AbstractModule {
     // pkg private for tests
     final Collection<AllocationDecider> deciderList;
     final ShardsAllocator shardsAllocator;
+    private final ClusterManagerMetrics clusterManagerMetrics;
 
     public ClusterModule(
         Settings settings,
@@ -166,6 +167,7 @@ public ClusterModule(
             settings,
             clusterManagerMetrics
         );
+        this.clusterManagerMetrics = clusterManagerMetrics;
     }
 
     public static List<Entry> getNamedWriteables() {
@@ -456,6 +458,7 @@ protected void configure() {
         bind(TaskResultsService.class).asEagerSingleton();
         bind(AllocationDeciders.class).toInstance(allocationDeciders);
         bind(ShardsAllocator.class).toInstance(shardsAllocator);
+        bind(ClusterManagerMetrics.class).toInstance(clusterManagerMetrics);
     }
 
     public void setExistingShardsAllocators(GatewayAllocator gatewayAllocator, ShardsBatchGatewayAllocator shardsBatchGatewayAllocator) {
diff --git a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java
index 1c12c260b3929..8ce11c8183cf6 100644
--- a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java
+++ b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java
@@ -103,10 +103,10 @@ public class NodeConnectionsService extends AbstractLifecycleComponent {
 
     // contains an entry for every node in the latest cluster state, as well as for nodes from which we are in the process of
     // disconnecting
-    private final Map<DiscoveryNode, ConnectionTarget> targetsByNode = new HashMap<>();
+    protected final Map<DiscoveryNode, ConnectionTarget> targetsByNode = new HashMap<>();
 
     private final TimeValue reconnectInterval;
-    private volatile ConnectionChecker connectionChecker;
+    protected volatile ConnectionChecker connectionChecker;
 
     @Inject
     public NodeConnectionsService(Settings settings, ThreadPool threadPool, TransportService transportService) {
@@ -115,6 +115,11 @@ public NodeConnectionsService(Settings settings, ThreadPool threadPool, Transpor
         this.reconnectInterval = NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(settings);
     }
 
+    // exposed for testing
+    protected ConnectionTarget createConnectionTarget(DiscoveryNode discoveryNode) {
+        return new ConnectionTarget(discoveryNode);
+    }
+
     /**
      * Connect to all the given nodes, but do not disconnect from any extra nodes. Calls the completion handler on completion of all
      * connection attempts to _new_ nodes, but not on attempts to re-establish connections to nodes that are already known.
@@ -159,6 +164,14 @@ public void connectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletion)
         runnables.forEach(Runnable::run);
     }
 
+    public void setPendingDisconnections(Set<DiscoveryNode> nodes) {
+        nodes.forEach(transportService::setPendingDisconnection);
+    }
+
+    public void clearPendingDisconnections() {
+        transportService.clearPendingDisconnections();
+    }
+
     /**
      * Disconnect from any nodes to which we are currently connected which do not appear in the given nodes. Does not wait for the
      * disconnections to complete, because they might have to wait for ongoing connection attempts first.
@@ -211,7 +224,7 @@ private void awaitPendingActivity(Runnable onCompletion) {
      * nodes which are in the process of disconnecting. The onCompletion handler is called after all ongoing connection/disconnection
      * attempts have completed.
      */
-    private void connectDisconnectedTargets(Runnable onCompletion) {
+    protected void connectDisconnectedTargets(Runnable onCompletion) {
         final List<Runnable> runnables = new ArrayList<>();
         synchronized (mutex) {
             final Collection<ConnectionTarget> connectionTargets = targetsByNode.values();
@@ -321,7 +334,7 @@ private enum ActivityType {
      *
      * @opensearch.internal
      */
-    private class ConnectionTarget {
+    protected class ConnectionTarget {
         private final DiscoveryNode discoveryNode;
 
         private PlainListenableActionFuture<Void> future = PlainListenableActionFuture.newListenableFuture();
diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java
index d658f38430dd9..e3f98a3f61534 100644
--- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java
+++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java
@@ -134,6 +134,38 @@ public static Entry startedEntry(
         );
     }
 
+    public static Entry startedEntry(
+        Snapshot snapshot,
+        boolean includeGlobalState,
+        boolean partial,
+        List<IndexId> indices,
+        List<String> dataStreams,
+        long startTime,
+        long repositoryStateId,
+        final Map<ShardId, ShardSnapshotStatus> shards,
+        Map<String, Object> userMetadata,
+        Version version,
+        boolean remoteStoreIndexShallowCopy,
+        boolean remoteStoreIndexShallowCopyV2
+    ) {
+        return new SnapshotsInProgress.Entry(
+            snapshot,
+            includeGlobalState,
+            partial,
+            completed(shards.values()) ? State.SUCCESS : State.STARTED,
+            indices,
+            dataStreams,
+            startTime,
+            repositoryStateId,
+            shards,
+            null,
+            userMetadata,
+            version,
+            remoteStoreIndexShallowCopy,
+            remoteStoreIndexShallowCopyV2
+        );
+    }
+
     /**
      * Creates the initial snapshot clone entry
      *
@@ -168,8 +200,39 @@ public static Entry startClone(
             version,
             source,
             Map.of(),
-            false // initialising to false, will be updated in startCloning method of SnapshotsService while updating entry with
-                  // clone jobs
+            false,
+            false// initialising to false, will be updated in startCloning method of SnapshotsService while updating entry with
+                 // clone jobs
+        );
+    }
+
+    public static Entry startClone(
+        Snapshot snapshot,
+        SnapshotId source,
+        List<IndexId> indices,
+        long startTime,
+        long repositoryStateId,
+        Version version,
+        boolean remoteStoreIndexShallowCopyV2
+    ) {
+        return new SnapshotsInProgress.Entry(
+            snapshot,
+            true,
+            false,
+            State.STARTED,
+            indices,
+            Collections.emptyList(),
+            startTime,
+            repositoryStateId,
+            Map.of(),
+            null,
+            Collections.emptyMap(),
+            version,
+            source,
+            Map.of(),
+            remoteStoreIndexShallowCopyV2,
+            remoteStoreIndexShallowCopyV2// initialising to false, will be updated in startCloning method of SnapshotsService
+            // while updating entry with clone jobs
         );
     }
 
@@ -183,6 +246,8 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation
         private final Snapshot snapshot;
         private final boolean includeGlobalState;
         private final boolean remoteStoreIndexShallowCopy;
+
+        private final boolean remoteStoreIndexShallowCopyV2;
         private final boolean partial;
         /**
          * Map of {@link ShardId} to {@link ShardSnapshotStatus} tracking the state of each shard snapshot operation.
@@ -212,6 +277,42 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation
         @Nullable
         private final String failure;
 
+        public Entry(
+            Snapshot snapshot,
+            boolean includeGlobalState,
+            boolean partial,
+            State state,
+            List<IndexId> indices,
+            List<String> dataStreams,
+            long startTime,
+            long repositoryStateId,
+            final Map<ShardId, ShardSnapshotStatus> shards,
+            String failure,
+            Map<String, Object> userMetadata,
+            Version version,
+            boolean remoteStoreIndexShallowCopy,
+            boolean remoteStoreIndexShallowCopyV2
+        ) {
+            this(
+                snapshot,
+                includeGlobalState,
+                partial,
+                state,
+                indices,
+                dataStreams,
+                startTime,
+                repositoryStateId,
+                shards,
+                failure,
+                userMetadata,
+                version,
+                null,
+                Map.of(),
+                remoteStoreIndexShallowCopy,
+                remoteStoreIndexShallowCopyV2
+            );
+        }
+
         // visible for testing, use #startedEntry and copy constructors in production code
         public Entry(
             Snapshot snapshot,
@@ -243,7 +344,8 @@ public Entry(
                 version,
                 null,
                 Map.of(),
-                remoteStoreIndexShallowCopy
+                remoteStoreIndexShallowCopy,
+                false
             );
         }
 
@@ -262,7 +364,8 @@ private Entry(
             Version version,
             @Nullable SnapshotId source,
             @Nullable final Map<RepositoryShardId, ShardSnapshotStatus> clones,
-            boolean remoteStoreIndexShallowCopy
+            boolean remoteStoreIndexShallowCopy,
+            boolean remoteStoreIndexShallowCopyV2
         ) {
             this.state = state;
             this.snapshot = snapshot;
@@ -284,7 +387,9 @@ private Entry(
                 this.clones = Collections.unmodifiableMap(clones);
             }
             this.remoteStoreIndexShallowCopy = remoteStoreIndexShallowCopy;
-            assert assertShardsConsistent(this.source, this.state, this.indices, this.shards, this.clones);
+            this.remoteStoreIndexShallowCopyV2 = remoteStoreIndexShallowCopyV2;
+            assert this.remoteStoreIndexShallowCopyV2
+                || assertShardsConsistent(this.source, this.state, this.indices, this.shards, this.clones);
         }
 
         private Entry(StreamInput in) throws IOException {
@@ -307,6 +412,11 @@ private Entry(StreamInput in) throws IOException {
             } else {
                 remoteStoreIndexShallowCopy = false;
             }
+            if (in.getVersion().onOrAfter(Version.V_2_18_0)) {
+                remoteStoreIndexShallowCopyV2 = in.readBoolean();
+            } else {
+                remoteStoreIndexShallowCopyV2 = false;
+            }
         }
 
         private static boolean assertShardsConsistent(
@@ -428,7 +538,8 @@ public Entry withRepoGen(long newRepoGen) {
                 version,
                 source,
                 clones,
-                remoteStoreIndexShallowCopy
+                remoteStoreIndexShallowCopy,
+                remoteStoreIndexShallowCopyV2
             );
         }
 
@@ -451,7 +562,8 @@ public Entry withClones(final Map<RepositoryShardId, ShardSnapshotStatus> update
                 version,
                 source,
                 updatedClones,
-                remoteStoreIndexShallowCopy
+                remoteStoreIndexShallowCopy,
+                remoteStoreIndexShallowCopyV2
             );
         }
 
@@ -471,7 +583,8 @@ public Entry withRemoteStoreIndexShallowCopy(final boolean remoteStoreIndexShall
                 version,
                 source,
                 clones,
-                remoteStoreIndexShallowCopy
+                remoteStoreIndexShallowCopy,
+                remoteStoreIndexShallowCopyV2
             );
         }
 
@@ -527,7 +640,8 @@ public Entry fail(final Map<ShardId, ShardSnapshotStatus> shards, State state, S
                 version,
                 source,
                 clones,
-                remoteStoreIndexShallowCopy
+                remoteStoreIndexShallowCopy,
+                remoteStoreIndexShallowCopyV2
             );
         }
 
@@ -614,6 +728,10 @@ public boolean remoteStoreIndexShallowCopy() {
             return remoteStoreIndexShallowCopy;
         }
 
+        public boolean remoteStoreIndexShallowCopyV2() {
+            return remoteStoreIndexShallowCopyV2;
+        }
+
         public Map<String, Object> userMetadata() {
             return userMetadata;
         }
@@ -678,6 +796,7 @@ public boolean equals(Object o) {
             if (Objects.equals(source, ((Entry) o).source) == false) return false;
             if (clones.equals(((Entry) o).clones) == false) return false;
             if (remoteStoreIndexShallowCopy != entry.remoteStoreIndexShallowCopy) return false;
+            if (remoteStoreIndexShallowCopyV2 != entry.remoteStoreIndexShallowCopyV2) return false;
             return true;
         }
 
@@ -695,6 +814,7 @@ public int hashCode() {
             result = 31 * result + (source == null ? 0 : source.hashCode());
             result = 31 * result + clones.hashCode();
             result = 31 * result + (remoteStoreIndexShallowCopy ? 1 : 0);
+            result = 31 * result + (remoteStoreIndexShallowCopyV2 ? 1 : 0);
             return result;
         }
 
@@ -766,6 +886,9 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getVersion().onOrAfter(Version.V_2_9_0)) {
                 out.writeBoolean(remoteStoreIndexShallowCopy);
             }
+            if (out.getVersion().onOrAfter(Version.V_2_18_0)) {
+                out.writeBoolean(remoteStoreIndexShallowCopyV2);
+            }
         }
 
         @Override
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java
index 9cffc7051d756..e5b07f867d609 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java
@@ -108,7 +108,7 @@ public CoordinationState(
         // ToDo: revisit this check while making the setting dynamic
         this.isRemotePublicationEnabled = isRemoteStateEnabled
             && REMOTE_PUBLICATION_SETTING.get(settings)
-            && localNode.isRemoteStatePublicationEnabled();
+            && localNode.isRemoteStatePublicationConfigured();
     }
 
     public boolean isRemotePublicationEnabled() {
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
index 13a57d93f03f0..1b3ae89251ac0 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
@@ -42,6 +42,7 @@
 import org.opensearch.cluster.ClusterStateTaskConfig;
 import org.opensearch.cluster.ClusterStateUpdateTask;
 import org.opensearch.cluster.LocalClusterUpdateTask;
+import org.opensearch.cluster.NodeConnectionsService;
 import org.opensearch.cluster.block.ClusterBlocks;
 import org.opensearch.cluster.coordination.ClusterFormationFailureHelper.ClusterFormationState;
 import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion;
@@ -187,6 +188,8 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery
     private final NodeHealthService nodeHealthService;
     private final PersistedStateRegistry persistedStateRegistry;
     private final RemoteStoreNodeService remoteStoreNodeService;
+    private NodeConnectionsService nodeConnectionsService;
+    private final RemoteClusterStateService remoteClusterStateService;
 
     /**
      * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}.
@@ -310,6 +313,7 @@ public Coordinator(
         this.persistedStateRegistry = persistedStateRegistry;
         this.localNodeCommissioned = true;
         this.remoteStoreNodeService = remoteStoreNodeService;
+        this.remoteClusterStateService = remoteClusterStateService;
     }
 
     private ClusterFormationState getClusterFormationState() {
@@ -418,7 +422,11 @@ PublishWithJoinResponse handlePublishRequest(PublishRequest publishRequest) {
 
         synchronized (mutex) {
             final DiscoveryNode sourceNode = publishRequest.getAcceptedState().nodes().getClusterManagerNode();
-            logger.trace("handlePublishRequest: handling [{}] from [{}]", publishRequest, sourceNode);
+            logger.debug(
+                "handlePublishRequest: handling version [{}] from [{}]",
+                publishRequest.getAcceptedState().getVersion(),
+                sourceNode
+            );
 
             if (sourceNode.equals(getLocalNode()) && mode != Mode.LEADER) {
                 // Rare case in which we stood down as leader between starting this publication and receiving it ourselves. The publication
@@ -630,7 +638,6 @@ private void handleJoinRequest(JoinRequest joinRequest, JoinHelper.JoinCallback
 
         transportService.connectToNode(joinRequest.getSourceNode(), ActionListener.wrap(ignore -> {
             final ClusterState stateForJoinValidation = getStateForClusterManagerService();
-
             if (stateForJoinValidation.nodes().isLocalNodeElectedClusterManager()) {
                 onJoinValidators.forEach(a -> a.accept(joinRequest.getSourceNode(), stateForJoinValidation));
                 if (stateForJoinValidation.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) == false) {
@@ -814,6 +821,10 @@ public void onFailure(String source, Exception e) {
             public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                 if (currentState.nodes().isLocalNodeElectedClusterManager() == false) {
                     allocationService.cleanCaches();
+                    // This set only needs to be maintained on active cluster-manager
+                    // This is cleaned up to avoid stale entries which would block future reconnections
+                    logger.trace("Removing all pending disconnections as part of cluster-manager cleanup");
+                    nodeConnectionsService.clearPendingDisconnections();
                 }
                 return unchanged();
             }
@@ -903,9 +914,9 @@ public DiscoveryStats stats() {
                 stats.add(persistedStateRegistry.getPersistedState(stateType).getStats());
             }
         });
-        if (coordinationState.get().isRemotePublicationEnabled()) {
-            stats.add(publicationHandler.getFullDownloadStats());
-            stats.add(publicationHandler.getDiffDownloadStats());
+        if (remoteClusterStateService != null) {
+            stats.add(remoteClusterStateService.getFullDownloadStats());
+            stats.add(remoteClusterStateService.getDiffDownloadStats());
         }
         clusterStateStats.setPersistenceStats(stats);
         return new DiscoveryStats(new PendingClusterStateStats(0, 0, 0), publicationHandler.stats(), clusterStateStats);
@@ -914,11 +925,18 @@ public DiscoveryStats stats() {
     @Override
     public void startInitialJoin() {
         synchronized (mutex) {
+            logger.trace("Starting initial join, becoming candidate");
             becomeCandidate("startInitialJoin");
         }
         clusterBootstrapService.scheduleUnconfiguredBootstrap();
     }
 
+    @Override
+    public void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) {
+        assert this.nodeConnectionsService == null : "nodeConnectionsService is already set";
+        this.nodeConnectionsService = nodeConnectionsService;
+    }
+
     @Override
     protected void doStop() {
         configuredHostsResolver.stop();
@@ -1356,6 +1374,9 @@ assert getLocalNode().equals(clusterState.getNodes().get(getLocalNode().getId())
                 currentPublication = Optional.of(publication);
 
                 final DiscoveryNodes publishNodes = publishRequest.getAcceptedState().nodes();
+                // marking pending disconnects before publish
+                // if a nodes tries to send a joinRequest while it is pending disconnect, it should fail
+                nodeConnectionsService.setPendingDisconnections(new HashSet<>(clusterChangedEvent.nodesDelta().removedNodes()));
                 leaderChecker.setCurrentNodes(publishNodes);
                 followersChecker.setCurrentNodes(publishNodes);
                 lagDetector.setTrackedNodes(publishNodes);
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java
index 838b5723b217b..aca8653be4dd8 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java
@@ -420,9 +420,14 @@ public static void ensureIndexCompatibility(final Version nodeVersion, Metadata
      * ensures that the joining node has a version that's compatible with all current nodes
      */
     public static void ensureNodesCompatibility(final DiscoveryNode joiningNode, DiscoveryNodes currentNodes, Metadata metadata) {
-        final Version minNodeVersion = currentNodes.getMinNodeVersion();
-        final Version maxNodeVersion = currentNodes.getMaxNodeVersion();
-        ensureNodesCompatibility(joiningNode, currentNodes, metadata, minNodeVersion, maxNodeVersion);
+        try {
+            final Version minNodeVersion = currentNodes.getMinNodeVersion();
+            final Version maxNodeVersion = currentNodes.getMaxNodeVersion();
+            ensureNodesCompatibility(joiningNode, currentNodes, metadata, minNodeVersion, maxNodeVersion);
+        } catch (Exception e) {
+            logger.error("Exception in NodesCompatibility validation", e);
+            throw e;
+        }
     }
 
     /**
@@ -508,10 +513,10 @@ private static void ensureRemoteClusterStateNodesCompatibility(DiscoveryNode joi
 
         assert existingNodes.isEmpty() == false;
         Optional<DiscoveryNode> remotePublicationNode = existingNodes.stream()
-            .filter(DiscoveryNode::isRemoteStatePublicationEnabled)
+            .filter(DiscoveryNode::isRemoteStatePublicationConfigured)
             .findFirst();
 
-        if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationEnabled()) {
+        if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationConfigured()) {
             ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES);
         }
     }
@@ -539,9 +544,11 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod
         CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings());
 
         List<String> reposToSkip = new ArrayList<>(1);
+        // find a remote node which has routing table configured
         Optional<DiscoveryNode> remoteRoutingTableNode = existingNodes.stream()
             .filter(
-                node -> node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) != null
+                node -> node.isRemoteStoreNode()
+                    && node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) != null
             )
             .findFirst();
         // If none of the existing nodes have routing table repo, then we skip this repo check if present in joining node.
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Publication.java b/server/src/main/java/org/opensearch/cluster/coordination/Publication.java
index 43801a05dbc24..3f7218939be92 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/Publication.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/Publication.java
@@ -85,7 +85,7 @@ public Publication(PublishRequest publishRequest, AckListener ackListener, LongS
     }
 
     public void start(Set<DiscoveryNode> faultyNodes) {
-        logger.trace("publishing {} to {}", publishRequest, publicationTargets);
+        logger.debug("publishing version {} to {}", publishRequest.getAcceptedState().getVersion(), publicationTargets);
 
         for (final DiscoveryNode faultyNode : faultyNodes) {
             onFaultyNode(faultyNode);
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java
index cdf331b7bb577..6277b70f9a7c8 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java
@@ -178,14 +178,6 @@ public PublishClusterStateStats stats() {
         );
     }
 
-    public PersistedStateStats getFullDownloadStats() {
-        return remoteClusterStateService.getFullDownloadStats();
-    }
-
-    public PersistedStateStats getDiffDownloadStats() {
-        return remoteClusterStateService.getDiffDownloadStats();
-    }
-
     private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportRequest request) throws IOException {
         try (StreamInput in = CompressedStreamUtils.decompressBytes(request, namedWriteableRegistry)) {
             ClusterState incomingState;
@@ -356,7 +348,7 @@ public PublicationContext newPublicationContext(
     ) {
         if (isRemotePublicationEnabled == true) {
             if (allNodesRemotePublicationEnabled.get() == false) {
-                if (validateRemotePublicationOnAllNodes(clusterChangedEvent.state().nodes()) == true) {
+                if (validateRemotePublicationConfiguredOnAllNodes(clusterChangedEvent.state().nodes()) == true) {
                     allNodesRemotePublicationEnabled.set(true);
                 }
             }
@@ -374,11 +366,11 @@ public PublicationContext newPublicationContext(
         return publicationContext;
     }
 
-    private boolean validateRemotePublicationOnAllNodes(DiscoveryNodes discoveryNodes) {
+    private boolean validateRemotePublicationConfiguredOnAllNodes(DiscoveryNodes discoveryNodes) {
         assert ClusterMetadataManifest.getCodecForVersion(discoveryNodes.getMinNodeVersion()) >= ClusterMetadataManifest.CODEC_V0;
         for (DiscoveryNode node : discoveryNodes.getNodes().values()) {
             // if a node is non-remote then created local publication context
-            if (node.isRemoteStatePublicationEnabled() == false) {
+            if (node.isRemoteStatePublicationConfigured() == false) {
                 return false;
             }
         }
@@ -542,7 +534,7 @@ public String executor() {
         }
 
         public void sendClusterState(DiscoveryNode destination, ActionListener<PublishWithJoinResponse> listener) {
-            logger.debug("sending cluster state over transport to node: {}", destination.getName());
+            logger.trace("sending cluster state over transport to node: {}", destination.getName());
             if (sendFullVersion || previousState.nodes().nodeExists(destination) == false) {
                 logger.trace("sending full cluster state version [{}] to [{}]", newState.version(), destination);
                 sendFullClusterState(destination, listener);
diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java
index a6f0a457f7f9b..b06c2ef0bdbe4 100644
--- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java
+++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java
@@ -515,10 +515,10 @@ public boolean isRemoteStoreNode() {
     }
 
     /**
-     * Returns whether remote cluster state publication is enabled on this node
+     * Returns whether settings required for remote cluster state publication is configured
      * @return true if the node contains remote cluster state node attribute and remote routing table node attribute
      */
-    public boolean isRemoteStatePublicationEnabled() {
+    public boolean isRemoteStatePublicationConfigured() {
         return this.getAttributes()
             .keySet()
             .stream()
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
index 47080cfbde692..d0b6f812e9ee2 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
@@ -502,6 +502,7 @@ private void runTask(UpdateTask task) {
             try {
                 applyChanges(task, previousClusterState, newClusterState, stopWatch);
                 TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, currentTimeInMillis() - startTimeMS));
+                // At this point, cluster state appliers and listeners are completed
                 logger.debug(
                     "processing [{}]: took [{}] done applying updated cluster state (version: {}, uuid: {})",
                     task.source,
@@ -510,6 +511,7 @@ private void runTask(UpdateTask task) {
                     newClusterState.stateUUID()
                 );
                 warnAboutSlowTaskIfNeeded(executionTime, task.source, stopWatch);
+                // Then we call the ClusterApplyListener of the task
                 task.listener.onSuccess(task.source);
             } catch (Exception e) {
                 TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, currentTimeInMillis() - startTimeMS));
@@ -578,6 +580,7 @@ private void applyChanges(UpdateTask task, ClusterState previousClusterState, Cl
 
         logger.debug("apply cluster state with version {}", newClusterState.version());
         callClusterStateAppliers(clusterChangedEvent, stopWatch);
+        logger.debug("completed calling appliers of cluster state for version {}", newClusterState.version());
 
         nodeConnectionsService.disconnectFromNodesExcept(newClusterState.nodes());
 
@@ -594,6 +597,7 @@ private void applyChanges(UpdateTask task, ClusterState previousClusterState, Cl
         state.set(newClusterState);
 
         callClusterStateListeners(clusterChangedEvent, stopWatch);
+        logger.debug("completed calling listeners of cluster state for version {}", newClusterState.version());
     }
 
     protected void connectToNodesAndWait(ClusterState newClusterState) {
diff --git a/server/src/main/java/org/opensearch/common/Table.java b/server/src/main/java/org/opensearch/common/Table.java
index da14f628efa0f..133ec3052e6c9 100644
--- a/server/src/main/java/org/opensearch/common/Table.java
+++ b/server/src/main/java/org/opensearch/common/Table.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.core.common.Strings;
+import org.opensearch.rest.pagination.PageToken;
 
 import java.time.Instant;
 import java.time.ZoneOffset;
@@ -59,9 +60,19 @@ public class Table {
     private List<Cell> currentCells;
     private boolean inHeaders = false;
     private boolean withTime = false;
+    /**
+     * paginatedQueryResponse if null will imply the Table response is not paginated.
+     */
+    private PageToken pageToken;
     public static final String EPOCH = "epoch";
     public static final String TIMESTAMP = "timestamp";
 
+    public Table() {}
+
+    public Table(@Nullable PageToken pageToken) {
+        this.pageToken = pageToken;
+    }
+
     public Table startHeaders() {
         inHeaders = true;
         currentCells = new ArrayList<>();
@@ -230,6 +241,10 @@ public Map<String, String> getAliasMap() {
         return headerAliasMap;
     }
 
+    public PageToken getPageToken() {
+        return pageToken;
+    }
+
     /**
      * Cell in a table
      *
diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java
index 97f304d776f5c..b769cdc2fe7ab 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java
@@ -14,6 +14,7 @@
 import org.opensearch.core.action.ActionListener;
 
 import java.io.IOException;
+import java.util.List;
 
 /**
  * An extension of {@link BlobContainer} that adds {@link AsyncMultiStreamBlobContainer#asyncBlobUpload} to allow
@@ -48,4 +49,8 @@ public interface AsyncMultiStreamBlobContainer extends BlobContainer {
      * by underlying blobContainer. In this case, caller doesn't need to ensure integrity of data.
      */
     boolean remoteIntegrityCheckSupported();
+
+    void deleteAsync(ActionListener<DeleteResult> completionListener);
+
+    void deleteBlobsAsyncIgnoringIfNotExists(List<String> blobNames, ActionListener<Void> completionListener);
 }
diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java
index 82bc7a0baed50..286c01f9dca44 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java
@@ -171,4 +171,14 @@ private InputStreamContainer decryptInputStreamContainer(InputStreamContainer in
             return new InputStreamContainer(decryptedStream, adjustedLength, adjustedPos);
         }
     }
+
+    @Override
+    public void deleteAsync(ActionListener<DeleteResult> completionListener) {
+        blobContainer.deleteAsync(completionListener);
+    }
+
+    @Override
+    public void deleteBlobsAsyncIgnoringIfNotExists(List<String> blobNames, ActionListener<Void> completionListener) {
+        blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, completionListener);
+    }
 }
diff --git a/server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java
new file mode 100644
index 0000000000000..db6785067edf6
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java
@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.breaker;
+
+import org.opensearch.OpenSearchException;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.rest.RestStatus;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Thrown when api response breaches threshold limit.
+ *
+ * @opensearch.internal
+ */
+public class ResponseLimitBreachedException extends OpenSearchException {
+
+    private final int responseLimit;
+    private final ResponseLimitSettings.LimitEntity limitEntity;
+
+    public ResponseLimitBreachedException(StreamInput in) throws IOException {
+        super(in);
+        responseLimit = in.readVInt();
+        limitEntity = in.readEnum(ResponseLimitSettings.LimitEntity.class);
+    }
+
+    public ResponseLimitBreachedException(String msg, int responseLimit, ResponseLimitSettings.LimitEntity limitEntity) {
+        super(msg);
+        this.responseLimit = responseLimit;
+        this.limitEntity = limitEntity;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeVInt(responseLimit);
+        out.writeEnum(limitEntity);
+    }
+
+    public int getResponseLimit() {
+        return responseLimit;
+    }
+
+    public ResponseLimitSettings.LimitEntity getLimitEntity() {
+        return limitEntity;
+    }
+
+    @Override
+    public RestStatus status() {
+        return RestStatus.TOO_MANY_REQUESTS;
+    }
+
+    @Override
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("response_limit", responseLimit);
+        builder.field("limit_entity", limitEntity);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java
new file mode 100644
index 0000000000000..8eb47c9aaf147
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java
@@ -0,0 +1,180 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.breaker;
+
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.routing.IndexRoutingTable;
+import org.opensearch.cluster.routing.IndexShardRoutingTable;
+import org.opensearch.cluster.routing.RoutingTable;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.rest.action.cat.RestIndicesAction;
+import org.opensearch.rest.action.cat.RestSegmentsAction;
+import org.opensearch.rest.action.cat.RestShardsAction;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Class to define dynamic settings for putting response limits on the actions and methods to evaluate if block is required.
+ */
+public class ResponseLimitSettings {
+
+    /**
+     * Enum to represent entity against which we need to perform limit checks.
+     */
+    public enum LimitEntity {
+        INDICES,
+        SHARDS
+    }
+
+    private volatile int catIndicesResponseLimit;
+    private volatile int catShardsResponseLimit;
+    private volatile int catSegmentsResponseLimit;
+
+    /**
+     * Setting to enable response limit on {@link RestIndicesAction}. The limit will be applied on number of indices.
+     */
+    public static final Setting<Integer> CAT_INDICES_RESPONSE_LIMIT_SETTING = Setting.intSetting(
+        "cat.indices.response.limit.number_of_indices",
+        -1,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * Setting to enable response limit on {@link RestShardsAction}. The limit will be applied on number of shards.
+     */
+    public static final Setting<Integer> CAT_SHARDS_RESPONSE_LIMIT_SETTING = Setting.intSetting(
+        "cat.shards.response.limit.number_of_shards",
+        -1,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * Setting to enable response limit on {@link RestSegmentsAction}. The limit will be applied on number of indices.
+     */
+    public static final Setting<Integer> CAT_SEGMENTS_RESPONSE_LIMIT_SETTING = Setting.intSetting(
+        "cat.segments.response.limit.number_of_indices",
+        -1,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    public ResponseLimitSettings(ClusterSettings clusterSettings, Settings settings) {
+        setCatShardsResponseLimit(CAT_SHARDS_RESPONSE_LIMIT_SETTING.get(settings));
+        setCatIndicesResponseLimit(CAT_INDICES_RESPONSE_LIMIT_SETTING.get(settings));
+        setCatSegmentsResponseLimit(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.get(settings));
+
+        clusterSettings.addSettingsUpdateConsumer(CAT_SHARDS_RESPONSE_LIMIT_SETTING, this::setCatShardsResponseLimit);
+        clusterSettings.addSettingsUpdateConsumer(CAT_INDICES_RESPONSE_LIMIT_SETTING, this::setCatIndicesResponseLimit);
+        clusterSettings.addSettingsUpdateConsumer(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING, this::setCatSegmentsResponseLimit);
+    }
+
+    /**
+     * Method to check if the response limit has reached for an action.
+     * The limits are controlled via dynamic settings.
+     *
+     * @param metadata     {@link Metadata}
+     * @param limitEntity  {@link LimitEntity}
+     * @param limit        Integer limit on block entity
+     * @return True/False
+     */
+    public static boolean isResponseLimitBreached(final Metadata metadata, final LimitEntity limitEntity, final int limit) {
+        if (Objects.isNull(metadata) || limit <= 0) return false;
+        if (limitEntity == LimitEntity.INDICES) {
+            int indicesCount = getTotalIndicesFromMetadata.apply(metadata);
+            return indicesCount > limit;
+        } else {
+            throw new IllegalArgumentException("Unsupported limit entity [" + limitEntity + "]");
+        }
+    }
+
+    /**
+     * Method to check if the response limit has reached for an action.
+     * The limits are controlled via dynamic settings.
+     *
+     * @param routingTable {@link RoutingTable}
+     * @param limitEntity  {@link LimitEntity}
+     * @param limit        Integer limit on block entity
+     * @return True/False
+     */
+    public static boolean isResponseLimitBreached(final RoutingTable routingTable, final LimitEntity limitEntity, final int limit) {
+        if (Objects.isNull(routingTable) || limit <= 0) return false;
+        if (Objects.isNull(limitEntity)) {
+            throw new IllegalArgumentException("Limit entity cannot be null");
+        }
+        switch (limitEntity) {
+            case INDICES:
+                int indicesCount = getTotalIndicesFromRoutingTable.apply(routingTable);
+                if (indicesCount > limit) return true;
+                break;
+            case SHARDS:
+                if (isShardsLimitBreached(routingTable, limit)) return true;
+                break;
+            default:
+                throw new IllegalArgumentException("Unsupported limit entity [" + limitEntity + "]");
+        }
+        return false;
+    }
+
+    private static boolean isShardsLimitBreached(final RoutingTable routingTable, final int limit) {
+        final Map<String, IndexRoutingTable> indexRoutingTableMap = routingTable.getIndicesRouting();
+        int totalShards = 0;
+        for (final Map.Entry<String, IndexRoutingTable> entry : indexRoutingTableMap.entrySet()) {
+            for (final Map.Entry<Integer, IndexShardRoutingTable> indexShardRoutingTableEntry : entry.getValue().getShards().entrySet()) {
+                totalShards += indexShardRoutingTableEntry.getValue().getShards().size();
+                // Fail fast if limit value is breached and avoid unnecessary computation.
+                if (totalShards > limit) return true;
+            }
+        }
+        return false;
+    }
+
+    private void setCatShardsResponseLimit(final int catShardsResponseLimit) {
+        this.catShardsResponseLimit = catShardsResponseLimit;
+    }
+
+    private void setCatIndicesResponseLimit(final int catIndicesResponseLimit) {
+        this.catIndicesResponseLimit = catIndicesResponseLimit;
+    }
+
+    private void setCatSegmentsResponseLimit(final int catSegmentsResponseLimit) {
+        this.catSegmentsResponseLimit = catSegmentsResponseLimit;
+    }
+
+    public int getCatShardsResponseLimit() {
+        return this.catShardsResponseLimit;
+    }
+
+    public int getCatIndicesResponseLimit() {
+        return this.catIndicesResponseLimit;
+    }
+
+    public int getCatSegmentsResponseLimit() {
+        return this.catSegmentsResponseLimit;
+    }
+
+    static Function<Metadata, Integer> getTotalIndicesFromMetadata = (metadata) -> {
+        if (Objects.nonNull(metadata) && Objects.nonNull(metadata.getIndices())) {
+            return metadata.getIndices().size();
+        }
+        return 0;
+    };
+
+    static Function<RoutingTable, Integer> getTotalIndicesFromRoutingTable = (routingTable) -> {
+        if (Objects.nonNull(routingTable) && Objects.nonNull(routingTable.getIndicesRouting())) {
+            return routingTable.getIndicesRouting().size();
+        }
+        return 0;
+    };
+}
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 09832e2b41b6d..e18c4dabb29eb 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -84,6 +84,7 @@
 import org.opensearch.cluster.service.ClusterManagerTaskThrottler;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.breaker.ResponseLimitSettings;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.settings.CacheSettings;
 import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings;
@@ -779,7 +780,10 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED,
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX,
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX,
+
+                // Snapshot related Settings
                 BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING,
+                BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING,
 
                 SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING,
 
@@ -793,7 +797,12 @@ public void apply(Settings value, Settings current, Settings previous) {
                 WorkloadManagementSettings.NODE_LEVEL_CPU_REJECTION_THRESHOLD,
                 WorkloadManagementSettings.NODE_LEVEL_CPU_CANCELLATION_THRESHOLD,
                 WorkloadManagementSettings.NODE_LEVEL_MEMORY_REJECTION_THRESHOLD,
-                WorkloadManagementSettings.NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD
+                WorkloadManagementSettings.NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD,
+
+                // Settings to be used for limiting rest requests
+                ResponseLimitSettings.CAT_INDICES_RESPONSE_LIMIT_SETTING,
+                ResponseLimitSettings.CAT_SHARDS_RESPONSE_LIMIT_SETTING,
+                ResponseLimitSettings.CAT_SEGMENTS_RESPONSE_LIMIT_SETTING
             )
         )
     );
diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
index 73375f9eaa413..59d999798868e 100644
--- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
@@ -31,7 +31,6 @@ protected FeatureFlagSettings(
 
     public static final Set<Setting<?>> BUILT_IN_FEATURE_FLAGS = Set.of(
         FeatureFlags.EXTENSIONS_SETTING,
-        FeatureFlags.IDENTITY_SETTING,
         FeatureFlags.TELEMETRY_SETTING,
         FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING,
         FeatureFlags.TIERED_REMOTE_INDEX_SETTING,
diff --git a/server/src/main/java/org/opensearch/common/settings/Setting.java b/server/src/main/java/org/opensearch/common/settings/Setting.java
index fea4c165809ba..081029c1c106c 100644
--- a/server/src/main/java/org/opensearch/common/settings/Setting.java
+++ b/server/src/main/java/org/opensearch/common/settings/Setting.java
@@ -1855,6 +1855,10 @@ public static Setting<Double> doubleSetting(
         );
     }
 
+    public static Setting<Double> doubleSetting(String key, double defaultValue, Validator<Double> validator, Property... properties) {
+        return new Setting<>(key, Double.toString(defaultValue), Double::parseDouble, validator, properties);
+    }
+
     /**
      * A writeable parser for double
      *
@@ -1961,6 +1965,15 @@ public static Setting<Double> doubleSetting(
         );
     }
 
+    public static Setting<Double> doubleSetting(
+        String key,
+        Setting<Double> fallbackSetting,
+        Validator<Double> validator,
+        Property... properties
+    ) {
+        return new Setting<>(new SimpleKey(key), fallbackSetting, fallbackSetting::getRaw, Double::parseDouble, validator, properties);
+    }
+
     /// simpleString
 
     public static Setting<String> simpleString(String key, Property... properties) {
diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
index a5acea004c3b2..6df68013a8119 100644
--- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
+++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
@@ -40,11 +40,6 @@ public class FeatureFlags {
      */
     public static final String EXTENSIONS = "opensearch.experimental.feature.extensions.enabled";
 
-    /**
-     * Gates the functionality of identity.
-     */
-    public static final String IDENTITY = "opensearch.experimental.feature.identity.enabled";
-
     /**
      * Gates the functionality of telemetry framework.
      */
@@ -82,8 +77,6 @@ public class FeatureFlags {
 
     public static final Setting<Boolean> EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope);
 
-    public static final Setting<Boolean> IDENTITY_SETTING = Setting.boolSetting(IDENTITY, false, Property.NodeScope);
-
     public static final Setting<Boolean> TELEMETRY_SETTING = Setting.boolSetting(TELEMETRY, false, Property.NodeScope);
 
     public static final Setting<Boolean> DATETIME_FORMATTER_CACHING_SETTING = Setting.boolSetting(
@@ -138,7 +131,6 @@ public class FeatureFlags {
     private static final List<Setting<Boolean>> ALL_FEATURE_FLAG_SETTINGS = List.of(
         REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
         EXTENSIONS_SETTING,
-        IDENTITY_SETTING,
         TELEMETRY_SETTING,
         DATETIME_FORMATTER_CACHING_SETTING,
         TIERED_REMOTE_INDEX_SETTING,
diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java
index 2f60fc8f69f87..95a8d9c9495f2 100644
--- a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java
+++ b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java
@@ -9,6 +9,7 @@
 package org.opensearch.common.xcontent;
 
 import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.common.ParsingException;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.xcontent.AbstractXContentParser;
@@ -73,7 +74,7 @@ public XContentParser parseObject() throws IOException {
         builder.startObject();
         LinkedList<String> path = new LinkedList<>(Collections.singleton(fieldTypeName));
         while (currentToken() != Token.END_OBJECT) {
-            parseToken(path, null);
+            parseToken(path);
         }
         // deduplication the fieldName,valueList,valueAndPathList
         builder.field(this.fieldTypeName, new HashSet<>(keyList));
@@ -87,14 +88,11 @@ public XContentParser parseObject() throws IOException {
     /**
      * @return true if the child object contains no_null value, false otherwise
      */
-    private boolean parseToken(Deque<String> path, String currentFieldName) throws IOException {
-        if (path.size() == 1 && processNoNestedValue()) {
-            return true;
-        }
+    private boolean parseToken(Deque<String> path) throws IOException {
         boolean isChildrenValueValid = false;
         boolean visitFieldName = false;
         if (this.parser.currentToken() == Token.FIELD_NAME) {
-            currentFieldName = this.parser.currentName();
+            final String currentFieldName = this.parser.currentName();
             path.addLast(currentFieldName); // Pushing onto the stack *must* be matched by pop
             visitFieldName = true;
             String parts = currentFieldName;
@@ -106,23 +104,21 @@ private boolean parseToken(Deque<String> path, String currentFieldName) throws I
             }
             this.keyList.add(parts); // parts has no dot, so either it's the original fieldName or it's the last part
             this.parser.nextToken(); // advance to the value of fieldName
-            isChildrenValueValid = parseToken(path, currentFieldName); // parse the value for fieldName (which will be an array, an object,
-                                                                       // or a primitive value)
+            isChildrenValueValid = parseToken(path); // parse the value for fieldName (which will be an array, an object,
+                                                     // or a primitive value)
             path.removeLast(); // Here is where we pop fieldName from the stack (since we're done with the value of fieldName)
             // Note that whichever other branch we just passed through has already ended with nextToken(), so we
             // don't need to call it.
         } else if (this.parser.currentToken() == Token.START_ARRAY) {
             parser.nextToken();
             while (this.parser.currentToken() != Token.END_ARRAY) {
-                isChildrenValueValid |= parseToken(path, currentFieldName);
+                isChildrenValueValid |= parseToken(path);
             }
             this.parser.nextToken();
-        } else if (this.parser.currentToken() == Token.END_ARRAY) {
-            // skip
         } else if (this.parser.currentToken() == Token.START_OBJECT) {
             parser.nextToken();
             while (this.parser.currentToken() != Token.END_OBJECT) {
-                isChildrenValueValid |= parseToken(path, currentFieldName);
+                isChildrenValueValid |= parseToken(path);
             }
             this.parser.nextToken();
         } else {
@@ -148,21 +144,6 @@ public void removeKeyOfNullValue() {
         this.keyList.remove(keyList.size() - 1);
     }
 
-    private boolean processNoNestedValue() throws IOException {
-        if (parser.currentToken() == Token.VALUE_NULL) {
-            return true;
-        } else if (this.parser.currentToken() == Token.VALUE_STRING
-            || this.parser.currentToken() == Token.VALUE_NUMBER
-            || this.parser.currentToken() == Token.VALUE_BOOLEAN) {
-                String value = this.parser.textOrNull();
-                if (value != null) {
-                    this.valueList.add(value);
-                }
-                return true;
-            }
-        return false;
-    }
-
     private String parseValue() throws IOException {
         switch (this.parser.currentToken()) {
             case VALUE_BOOLEAN:
@@ -172,7 +153,7 @@ private String parseValue() throws IOException {
                 return this.parser.textOrNull();
             // Handle other token types as needed
             default:
-                throw new IOException("Unsupported value token type [" + parser.currentToken() + "]");
+                throw new ParsingException(parser.getTokenLocation(), "Unexpected value token type [" + parser.currentToken() + "]");
         }
     }
 
diff --git a/server/src/main/java/org/opensearch/discovery/Discovery.java b/server/src/main/java/org/opensearch/discovery/Discovery.java
index 9d6807b6522c9..6d9fb1f4985df 100644
--- a/server/src/main/java/org/opensearch/discovery/Discovery.java
+++ b/server/src/main/java/org/opensearch/discovery/Discovery.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.discovery;
 
+import org.opensearch.cluster.NodeConnectionsService;
 import org.opensearch.cluster.coordination.ClusterStatePublisher;
 import org.opensearch.common.lifecycle.LifecycleComponent;
 
@@ -54,4 +55,8 @@ public interface Discovery extends LifecycleComponent, ClusterStatePublisher {
      */
     void startInitialJoin();
 
+    /**
+     * Sets the NodeConnectionsService which is an abstraction used for connection management
+     */
+    void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService);
 }
diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java
index df642a9f5a743..d86d41bb1a359 100644
--- a/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java
+++ b/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.support.nodes.BaseNodeResponse;
 import org.opensearch.action.support.nodes.BaseNodesResponse;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.common.logging.Loggers;
 import org.opensearch.core.index.shard.ShardId;
@@ -48,7 +49,8 @@ public abstract class AsyncShardBatchFetch<T extends BaseNodeResponse, V> extend
         Class<V> clazz,
         V emptyShardResponse,
         Predicate<V> emptyShardResponsePredicate,
-        ShardBatchResponseFactory<T, V> responseFactory
+        ShardBatchResponseFactory<T, V> responseFactory,
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
         super(
             logger,
@@ -64,7 +66,8 @@ public abstract class AsyncShardBatchFetch<T extends BaseNodeResponse, V> extend
                 clazz,
                 emptyShardResponse,
                 emptyShardResponsePredicate,
-                responseFactory
+                responseFactory,
+                clusterManagerMetrics
             )
         );
     }
@@ -116,9 +119,10 @@ public ShardBatchCache(
             Class<V> clazz,
             V emptyResponse,
             Predicate<V> emptyShardResponsePredicate,
-            ShardBatchResponseFactory<T, V> responseFactory
+            ShardBatchResponseFactory<T, V> responseFactory,
+            ClusterManagerMetrics clusterManagerMetrics
         ) {
-            super(Loggers.getLogger(logger, "_" + logKey), type);
+            super(Loggers.getLogger(logger, "_" + logKey), type, clusterManagerMetrics);
             this.batchSize = shardAttributesMap.size();
             this.emptyShardResponsePredicate = emptyShardResponsePredicate;
             cache = new HashMap<>();
diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java
index b664dd573ce67..6017743ef2bd0 100644
--- a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java
+++ b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.FailedNodeException;
 import org.opensearch.action.support.nodes.BaseNodeResponse;
 import org.opensearch.action.support.nodes.BaseNodesResponse;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.allocation.RoutingAllocation;
@@ -94,7 +95,8 @@ protected AsyncShardFetch(
         String type,
         ShardId shardId,
         String customDataPath,
-        Lister<? extends BaseNodesResponse<T>, T> action
+        Lister<? extends BaseNodesResponse<T>, T> action,
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
         this.logger = logger;
         this.type = type;
@@ -102,7 +104,7 @@ protected AsyncShardFetch(
         shardAttributesMap.put(shardId, new ShardAttributes(customDataPath));
         this.action = (Lister<BaseNodesResponse<T>, T>) action;
         this.reroutingKey = "ShardId=[" + shardId.toString() + "]";
-        cache = new ShardCache<>(logger, reroutingKey, type);
+        cache = new ShardCache<>(logger, reroutingKey, type, clusterManagerMetrics);
     }
 
     /**
@@ -284,8 +286,8 @@ static class ShardCache<K extends BaseNodeResponse> extends AsyncShardFetchCache
 
         private final Map<String, NodeEntry<K>> cache;
 
-        public ShardCache(Logger logger, String logKey, String type) {
-            super(Loggers.getLogger(logger, "_" + logKey), type);
+        public ShardCache(Logger logger, String logKey, String type, ClusterManagerMetrics clusterManagerMetrics) {
+            super(Loggers.getLogger(logger, "_" + logKey), type, clusterManagerMetrics);
             cache = new HashMap<>();
         }
 
diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java b/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java
index 2a4e6181467b0..9b0a95f611e0e 100644
--- a/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java
+++ b/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java
@@ -14,6 +14,7 @@
 import org.opensearch.OpenSearchTimeoutException;
 import org.opensearch.action.FailedNodeException;
 import org.opensearch.action.support.nodes.BaseNodeResponse;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
@@ -51,10 +52,12 @@ public abstract class AsyncShardFetchCache<K extends BaseNodeResponse> {
 
     private final Logger logger;
     private final String type;
+    private final ClusterManagerMetrics clusterManagerMetrics;
 
-    protected AsyncShardFetchCache(Logger logger, String type) {
+    protected AsyncShardFetchCache(Logger logger, String type, ClusterManagerMetrics clusterManagerMetrics) {
         this.logger = logger;
         this.type = type;
+        this.clusterManagerMetrics = clusterManagerMetrics;
     }
 
     abstract void initData(DiscoveryNode node);
@@ -162,6 +165,7 @@ Map<DiscoveryNode, K> getCacheData(DiscoveryNodes nodes, Set<String> failedNodes
     }
 
     void processResponses(List<K> responses, long fetchingRound) {
+        clusterManagerMetrics.incrementCounter(clusterManagerMetrics.asyncFetchSuccessCounter, Double.valueOf(responses.size()));
         for (K response : responses) {
             BaseNodeEntry nodeEntry = getCache().get(response.getNode().getId());
             if (nodeEntry != null) {
@@ -222,6 +226,7 @@ boolean retryableException(Throwable unwrappedCause) {
     }
 
     void processFailures(List<FailedNodeException> failures, long fetchingRound) {
+        clusterManagerMetrics.incrementCounter(clusterManagerMetrics.asyncFetchFailureCounter, Double.valueOf(failures.size()));
         for (FailedNodeException failure : failures) {
             logger.trace("processing failure {} for [{}]", failure, type);
             BaseNodeEntry nodeEntry = getCache().get(failure.nodeId());
diff --git a/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java
index c8ef9364ebba9..eaacb5dbfbd17 100644
--- a/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java
@@ -37,6 +37,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.action.support.nodes.BaseNodeResponse;
 import org.opensearch.action.support.nodes.BaseNodesResponse;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
@@ -92,11 +93,12 @@ public class GatewayAllocator implements ExistingShardsAllocator {
     public GatewayAllocator(
         RerouteService rerouteService,
         TransportNodesListGatewayStartedShards startedAction,
-        TransportNodesListShardStoreMetadata storeAction
+        TransportNodesListShardStoreMetadata storeAction,
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
         this.rerouteService = rerouteService;
-        this.primaryShardAllocator = new InternalPrimaryShardAllocator(startedAction);
-        this.replicaShardAllocator = new InternalReplicaShardAllocator(storeAction);
+        this.primaryShardAllocator = new InternalPrimaryShardAllocator(startedAction, clusterManagerMetrics);
+        this.replicaShardAllocator = new InternalReplicaShardAllocator(storeAction, clusterManagerMetrics);
     }
 
     @Override
@@ -251,9 +253,10 @@ class InternalAsyncFetch<T extends BaseNodeResponse> extends AsyncShardFetch<T>
             String type,
             ShardId shardId,
             String customDataPath,
-            Lister<? extends BaseNodesResponse<T>, T> action
+            Lister<? extends BaseNodesResponse<T>, T> action,
+            ClusterManagerMetrics clusterManagerMetrics
         ) {
-            super(logger, type, shardId, customDataPath, action);
+            super(logger, type, shardId, customDataPath, action, clusterManagerMetrics);
         }
 
         @Override
@@ -274,9 +277,11 @@ protected void reroute(String reroutingKey, String reason) {
     class InternalPrimaryShardAllocator extends PrimaryShardAllocator {
 
         private final TransportNodesListGatewayStartedShards startedAction;
+        private final ClusterManagerMetrics clusterManagerMetrics;
 
-        InternalPrimaryShardAllocator(TransportNodesListGatewayStartedShards startedAction) {
+        InternalPrimaryShardAllocator(TransportNodesListGatewayStartedShards startedAction, ClusterManagerMetrics clusterManagerMetrics) {
             this.startedAction = startedAction;
+            this.clusterManagerMetrics = clusterManagerMetrics;
         }
 
         @Override
@@ -291,7 +296,8 @@ protected AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShards.Nod
                     "shard_started",
                     shardId,
                     IndexMetadata.INDEX_DATA_PATH_SETTING.get(allocation.metadata().index(shard.index()).getSettings()),
-                    startedAction
+                    startedAction,
+                    clusterManagerMetrics
                 )
             );
             AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> shardState = fetch.fetchData(
@@ -313,9 +319,11 @@ protected AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShards.Nod
     class InternalReplicaShardAllocator extends ReplicaShardAllocator {
 
         private final TransportNodesListShardStoreMetadata storeAction;
+        private final ClusterManagerMetrics clusterManagerMetrics;
 
-        InternalReplicaShardAllocator(TransportNodesListShardStoreMetadata storeAction) {
+        InternalReplicaShardAllocator(TransportNodesListShardStoreMetadata storeAction, ClusterManagerMetrics clusterManagerMetrics) {
             this.storeAction = storeAction;
+            this.clusterManagerMetrics = clusterManagerMetrics;
         }
 
         @Override
@@ -330,7 +338,8 @@ protected AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetadata.NodeS
                     "shard_store",
                     shard.shardId(),
                     IndexMetadata.INDEX_DATA_PATH_SETTING.get(allocation.metadata().index(shard.index()).getSettings()),
-                    storeAction
+                    storeAction,
+                    clusterManagerMetrics
                 )
             );
             AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetadata.NodeStoreFilesMetadata> shardStores = fetch.fetchData(
diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
index b3836edcd7d6c..f5da6df2689bd 100644
--- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
@@ -66,13 +66,13 @@
 import org.opensearch.gateway.remote.RemoteClusterStateService;
 import org.opensearch.gateway.remote.model.RemoteClusterStateManifestInfo;
 import org.opensearch.index.recovery.RemoteStoreRestoreService;
-import org.opensearch.index.recovery.RemoteStoreRestoreService.RemoteRestoreResult;
 import org.opensearch.node.Node;
 import org.opensearch.plugins.MetadataUpgrader;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
 import java.io.Closeable;
+import java.io.IOError;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Collections;
@@ -109,6 +109,8 @@ public class GatewayMetaState implements Closeable {
      */
     public static final String STALE_STATE_CONFIG_NODE_ID = "STALE_STATE_CONFIG";
 
+    private final Logger logger = LogManager.getLogger(GatewayMetaState.class);
+
     private PersistedStateRegistry persistedStateRegistry;
 
     public PersistedState getPersistedState() {
@@ -175,15 +177,11 @@ public void start(
                             );
                             if (ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID) == false) {
                                 // Load state from remote
-                                final RemoteRestoreResult remoteRestoreResult = remoteStoreRestoreService.restore(
-                                    // Remote Metadata should always override local disk Metadata
-                                    // if local disk Metadata's cluster uuid is UNKNOWN_UUID
-                                    ClusterState.builder(clusterState).metadata(Metadata.EMPTY_METADATA).build(),
-                                    lastKnownClusterUUID,
-                                    false,
-                                    new String[] {}
+                                clusterState = restoreClusterStateWithRetries(
+                                    remoteStoreRestoreService,
+                                    clusterState,
+                                    lastKnownClusterUUID
                                 );
-                                clusterState = remoteRestoreResult.getClusterState();
                             }
                         }
                         remotePersistedState = new RemotePersistedState(remoteClusterStateService, lastKnownClusterUUID);
@@ -258,6 +256,50 @@ public void start(
         }
     }
 
+    private ClusterState restoreClusterStateWithRetries(
+        RemoteStoreRestoreService remoteStoreRestoreService,
+        ClusterState clusterState,
+        String lastKnownClusterUUID
+    ) {
+        int maxAttempts = 5;
+        int delayInMills = 200;
+        for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+            try {
+                logger.info("Attempt {} to restore cluster state", attempt);
+                return restoreClusterState(remoteStoreRestoreService, clusterState, lastKnownClusterUUID);
+            } catch (Exception e) {
+                if (attempt == maxAttempts) {
+                    // Throw an Error so that the process is halted.
+                    throw new IOError(e);
+                }
+                try {
+                    TimeUnit.MILLISECONDS.sleep(delayInMills);
+                } catch (InterruptedException ie) {
+                    Thread.currentThread().interrupt(); // Restore interrupted status
+                    throw new RuntimeException(ie);
+                }
+                delayInMills = delayInMills * 2;
+            }
+        }
+        // This statement will never be reached.
+        return null;
+    }
+
+    ClusterState restoreClusterState(
+        RemoteStoreRestoreService remoteStoreRestoreService,
+        ClusterState clusterState,
+        String lastKnownClusterUUID
+    ) {
+        return remoteStoreRestoreService.restore(
+            // Remote Metadata should always override local disk Metadata
+            // if local disk Metadata's cluster uuid is UNKNOWN_UUID
+            ClusterState.builder(clusterState).metadata(Metadata.EMPTY_METADATA).build(),
+            lastKnownClusterUUID,
+            false,
+            new String[] {}
+        ).getClusterState();
+    }
+
     // exposed so it can be overridden by tests
     ClusterState prepareInitialClusterState(TransportService transportService, ClusterService clusterService, ClusterState clusterState) {
         assert clusterState.nodes().getLocalNode() == null : "prepareInitialClusterState must only be called once";
diff --git a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java
index 5e2dcbcd70b40..d7c0a66ba3424 100644
--- a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java
@@ -13,6 +13,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.action.support.nodes.BaseNodeResponse;
 import org.opensearch.action.support.nodes.BaseNodesResponse;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
@@ -44,6 +45,7 @@
 import org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadata;
 import org.opensearch.indices.store.TransportNodesListShardStoreMetadataHelper;
 import org.opensearch.indices.store.TransportNodesListShardStoreMetadataHelper.StoreFilesMetadata;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -81,6 +83,7 @@ public class ShardsBatchGatewayAllocator implements ExistingShardsAllocator {
     private TimeValue primaryShardsBatchGatewayAllocatorTimeout;
     private TimeValue replicaShardsBatchGatewayAllocatorTimeout;
     public static final TimeValue MIN_ALLOCATOR_TIMEOUT = TimeValue.timeValueSeconds(20);
+    private final ClusterManagerMetrics clusterManagerMetrics;
 
     /**
      * Number of shards we send in one batch to data nodes for fetching metadata
@@ -160,7 +163,8 @@ public ShardsBatchGatewayAllocator(
         TransportNodesListGatewayStartedShardsBatch batchStartedAction,
         TransportNodesListShardStoreMetadataBatch batchStoreAction,
         Settings settings,
-        ClusterSettings clusterSettings
+        ClusterSettings clusterSettings,
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
         this.rerouteService = rerouteService;
         this.primaryShardBatchAllocator = new InternalPrimaryBatchShardAllocator();
@@ -172,6 +176,7 @@ public ShardsBatchGatewayAllocator(
         clusterSettings.addSettingsUpdateConsumer(PRIMARY_BATCH_ALLOCATOR_TIMEOUT_SETTING, this::setPrimaryBatchAllocatorTimeout);
         this.replicaShardsBatchGatewayAllocatorTimeout = REPLICA_BATCH_ALLOCATOR_TIMEOUT_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(REPLICA_BATCH_ALLOCATOR_TIMEOUT_SETTING, this::setReplicaBatchAllocatorTimeout);
+        this.clusterManagerMetrics = clusterManagerMetrics;
     }
 
     @Override
@@ -187,6 +192,7 @@ protected ShardsBatchGatewayAllocator() {
         this(DEFAULT_SHARD_BATCH_SIZE, null);
     }
 
+    // for tests
     protected ShardsBatchGatewayAllocator(long batchSize, RerouteService rerouteService) {
         this.rerouteService = rerouteService;
         this.batchStartedAction = null;
@@ -196,10 +202,9 @@ protected ShardsBatchGatewayAllocator(long batchSize, RerouteService rerouteServ
         this.maxBatchSize = batchSize;
         this.primaryShardsBatchGatewayAllocatorTimeout = null;
         this.replicaShardsBatchGatewayAllocatorTimeout = null;
+        this.clusterManagerMetrics = new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE);
     }
 
-    // for tests
-
     @Override
     public int getNumberOfInFlightFetches() {
         int count = 0;
@@ -413,7 +418,7 @@ else if (shardRouting.primary() == primary) {
             // add to batch if batch size full or last shard in unassigned list
             if (batchSize == 0 || iterator.hasNext() == false) {
                 String batchUUId = UUIDs.base64UUID();
-                ShardsBatch shardsBatch = new ShardsBatch(batchUUId, perBatchShards, primary);
+                ShardsBatch shardsBatch = new ShardsBatch(batchUUId, perBatchShards, primary, clusterManagerMetrics);
                 // add the batch to list of current batches
                 addBatch(shardsBatch, primary);
                 batchesToBeAssigned.add(batchUUId);
@@ -588,9 +593,21 @@ class InternalBatchAsyncFetch<T extends BaseNodeResponse, V> extends AsyncShardB
             Class<V> clazz,
             V emptyShardResponse,
             Predicate<V> emptyShardResponsePredicate,
-            ShardBatchResponseFactory<T, V> responseFactory
+            ShardBatchResponseFactory<T, V> responseFactory,
+            ClusterManagerMetrics clusterManagerMetrics
         ) {
-            super(logger, type, map, action, batchUUId, clazz, emptyShardResponse, emptyShardResponsePredicate, responseFactory);
+            super(
+                logger,
+                type,
+                map,
+                action,
+                batchUUId,
+                clazz,
+                emptyShardResponse,
+                emptyShardResponsePredicate,
+                responseFactory,
+                clusterManagerMetrics
+            );
         }
 
         @Override
@@ -650,16 +667,17 @@ protected boolean hasInitiatedFetching(ShardRouting shard) {
              * It should return false if there has never been a fetch for this batch.
              * This function is currently only used in the case of replica shards when all deciders returned NO/THROTTLE, and explain mode is ON.
              * Allocation explain and manual reroute APIs try to append shard store information (matching bytes) to the allocation decision.
-             * However, these APIs do not want to trigger a new asyncFetch for these ineligible shards, unless the data from nodes is already there.
+             * However, these APIs do not want to trigger a new asyncFetch for these ineligible shards
+             * They only want to use the data if it is already available.
              * This function is used to see if a fetch has happened to decide if it is possible to append shard store info without a new async fetch.
              * In the case when shard has a batch but no fetch has happened before, it would be because it is a new batch.
              * In the case when shard has a batch, and a fetch has happened before, and no fetch is ongoing, it would be because we have already completed fetch for all nodes.
-             *
+             * <p>
              * In order to check if a fetch has ever happened, we check 2 things:
              * 1. If the shard batch cache is empty, we know that fetch has never happened so we return false.
              * 2. If we see that the list of nodes to fetch from is empty, we know that all nodes have data or are ongoing a fetch. So we return true.
              * 3. Otherwise we return false.
-             *
+             * <p>
              * see {@link AsyncShardFetchCache#findNodesToFetch()}
              */
             String batchId = getBatchId(shard, shard.primary());
@@ -669,7 +687,8 @@ protected boolean hasInitiatedFetching(ShardRouting shard) {
             logger.trace("Checking if fetching done for batch id {}", batchId);
             ShardsBatch shardsBatch = shard.primary() ? batchIdToStartedShardBatch.get(batchId) : batchIdToStoreShardBatch.get(batchId);
             // if fetchData has never been called, the per node cache will be empty and have no nodes
-            // this is because cache.fillShardCacheWithDataNodes(nodes) initialises this map and is called in AsyncShardFetch.fetchData
+            /// this is because {@link AsyncShardFetchCache#fillShardCacheWithDataNodes(DiscoveryNodes)} initialises this map
+            /// and is called in {@link AsyncShardFetch#fetchData(DiscoveryNodes, Map)}
             if (shardsBatch == null || shardsBatch.getAsyncFetcher().hasEmptyCache()) {
                 logger.trace("Batch cache is empty for batch {} ", batchId);
                 return false;
@@ -739,7 +758,12 @@ public class ShardsBatch {
 
         private final Map<ShardId, ShardEntry> batchInfo;
 
-        public ShardsBatch(String batchId, Map<ShardId, ShardEntry> shardsWithInfo, boolean primary) {
+        public ShardsBatch(
+            String batchId,
+            Map<ShardId, ShardEntry> shardsWithInfo,
+            boolean primary,
+            ClusterManagerMetrics clusterManagerMetrics
+        ) {
             this.batchId = batchId;
             this.batchInfo = new HashMap<>(shardsWithInfo);
             // create a ShardId -> customDataPath map for async fetch
@@ -757,7 +781,8 @@ public ShardsBatch(String batchId, Map<ShardId, ShardEntry> shardsWithInfo, bool
                     GatewayStartedShard.class,
                     new GatewayStartedShard(null, false, null, null),
                     GatewayStartedShard::isEmpty,
-                    new ShardBatchResponseFactory<>(true)
+                    new ShardBatchResponseFactory<>(true),
+                    clusterManagerMetrics
                 );
             } else {
                 asyncBatch = new InternalBatchAsyncFetch<>(
@@ -769,7 +794,8 @@ public ShardsBatch(String batchId, Map<ShardId, ShardEntry> shardsWithInfo, bool
                     NodeStoreFilesMetadata.class,
                     new NodeStoreFilesMetadata(new StoreFilesMetadata(null, Store.MetadataSnapshot.EMPTY, Collections.emptyList()), null),
                     NodeStoreFilesMetadata::isEmpty,
-                    new ShardBatchResponseFactory<>(false)
+                    new ShardBatchResponseFactory<>(false),
+                    clusterManagerMetrics
                 );
             }
         }
diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java
index d6739c4572d1a..aa007f5da15b3 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java
@@ -12,8 +12,10 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.DiffableStringMap;
+import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -22,11 +24,15 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParseException;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
 
 import com.jcraft.jzlib.JZlib;
 
@@ -37,6 +43,7 @@
  */
 public class ClusterStateChecksum implements ToXContentFragment, Writeable {
 
+    public static final int COMPONENT_SIZE = 11;
     static final String ROUTING_TABLE_CS = "routing_table";
     static final String NODES_CS = "discovery_nodes";
     static final String BLOCKS_CS = "blocks";
@@ -65,62 +72,103 @@ public class ClusterStateChecksum implements ToXContentFragment, Writeable {
     long indicesChecksum;
     long clusterStateChecksum;
 
-    public ClusterStateChecksum(ClusterState clusterState) {
-        try (
-            BytesStreamOutput out = new BytesStreamOutput();
-            BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out)
-        ) {
-            clusterState.routingTable().writeVerifiableTo(checksumOut);
-            routingTableChecksum = checksumOut.getChecksum();
-
-            checksumOut.reset();
-            clusterState.nodes().writeVerifiableTo(checksumOut);
-            nodesChecksum = checksumOut.getChecksum();
-
-            checksumOut.reset();
-            clusterState.coordinationMetadata().writeVerifiableTo(checksumOut);
-            coordinationMetadataChecksum = checksumOut.getChecksum();
-
-            // Settings create sortedMap by default, so no explicit sorting required here.
-            checksumOut.reset();
-            Settings.writeSettingsToStream(clusterState.metadata().persistentSettings(), checksumOut);
-            settingMetadataChecksum = checksumOut.getChecksum();
-
-            checksumOut.reset();
-            Settings.writeSettingsToStream(clusterState.metadata().transientSettings(), checksumOut);
-            transientSettingsMetadataChecksum = checksumOut.getChecksum();
-
-            checksumOut.reset();
-            clusterState.metadata().templatesMetadata().writeVerifiableTo(checksumOut);
-            templatesMetadataChecksum = checksumOut.getChecksum();
-
-            checksumOut.reset();
-            checksumOut.writeStringCollection(clusterState.metadata().customs().keySet());
-            customMetadataMapChecksum = checksumOut.getChecksum();
-
-            checksumOut.reset();
-            ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeTo(checksumOut);
-            hashesOfConsistentSettingsChecksum = checksumOut.getChecksum();
-
-            checksumOut.reset();
-            checksumOut.writeMapValues(
+    public ClusterStateChecksum(ClusterState clusterState, ThreadPool threadpool) {
+        long start = threadpool.relativeTimeInNanos();
+        ExecutorService executorService = threadpool.executor(ThreadPool.Names.REMOTE_STATE_CHECKSUM);
+        CountDownLatch latch = new CountDownLatch(COMPONENT_SIZE);
+
+        executeChecksumTask((stream) -> {
+            clusterState.routingTable().writeVerifiableTo(stream);
+            return null;
+        }, checksum -> routingTableChecksum = checksum, executorService, latch);
+
+        executeChecksumTask((stream) -> {
+            clusterState.nodes().writeVerifiableTo(stream);
+            return null;
+        }, checksum -> nodesChecksum = checksum, executorService, latch);
+
+        executeChecksumTask((stream) -> {
+            clusterState.coordinationMetadata().writeVerifiableTo(stream);
+            return null;
+        }, checksum -> coordinationMetadataChecksum = checksum, executorService, latch);
+
+        executeChecksumTask((stream) -> {
+            Settings.writeSettingsToStream(clusterState.metadata().persistentSettings(), stream);
+            return null;
+        }, checksum -> settingMetadataChecksum = checksum, executorService, latch);
+
+        executeChecksumTask((stream) -> {
+            Settings.writeSettingsToStream(clusterState.metadata().transientSettings(), stream);
+            return null;
+        }, checksum -> transientSettingsMetadataChecksum = checksum, executorService, latch);
+
+        executeChecksumTask((stream) -> {
+            clusterState.metadata().templatesMetadata().writeVerifiableTo(stream);
+            return null;
+        }, checksum -> templatesMetadataChecksum = checksum, executorService, latch);
+
+        executeChecksumTask((stream) -> {
+            stream.writeStringCollection(clusterState.metadata().customs().keySet());
+            return null;
+        }, checksum -> customMetadataMapChecksum = checksum, executorService, latch);
+
+        executeChecksumTask((stream) -> {
+            ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeTo(stream);
+            return null;
+        }, checksum -> hashesOfConsistentSettingsChecksum = checksum, executorService, latch);
+
+        executeChecksumTask((stream) -> {
+            stream.writeMapValues(
                 clusterState.metadata().indices(),
-                (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)
+                (checksumStream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) checksumStream)
             );
-            indicesChecksum = checksumOut.getChecksum();
-
-            checksumOut.reset();
-            clusterState.blocks().writeVerifiableTo(checksumOut);
-            blocksChecksum = checksumOut.getChecksum();
-
-            checksumOut.reset();
-            checksumOut.writeStringCollection(clusterState.customs().keySet());
-            clusterStateCustomsChecksum = checksumOut.getChecksum();
-        } catch (IOException e) {
-            logger.error("Failed to create checksum for cluster state.", e);
+            return null;
+        }, checksum -> indicesChecksum = checksum, executorService, latch);
+
+        executeChecksumTask((stream) -> {
+            clusterState.blocks().writeVerifiableTo(stream);
+            return null;
+        }, checksum -> blocksChecksum = checksum, executorService, latch);
+
+        executeChecksumTask((stream) -> {
+            stream.writeStringCollection(clusterState.customs().keySet());
+            return null;
+        }, checksum -> clusterStateCustomsChecksum = checksum, executorService, latch);
+
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
             throw new RemoteStateTransferException("Failed to create checksum for cluster state.", e);
         }
         createClusterStateChecksum();
+        logger.debug("Checksum execution time {}", TimeValue.nsecToMSec(threadpool.relativeTimeInNanos() - start));
+    }
+
+    private void executeChecksumTask(
+        CheckedFunction<BufferedChecksumStreamOutput, Void, IOException> checksumTask,
+        Consumer<Long> checksumConsumer,
+        ExecutorService executorService,
+        CountDownLatch latch
+    ) {
+        executorService.execute(() -> {
+            try {
+                long checksum = createChecksum(checksumTask);
+                checksumConsumer.accept(checksum);
+                latch.countDown();
+            } catch (IOException e) {
+                throw new RemoteStateTransferException("Failed to execute checksum task", e);
+            }
+        });
+    }
+
+    private long createChecksum(CheckedFunction<BufferedChecksumStreamOutput, Void, IOException> task) throws IOException {
+        try (
+            BytesStreamOutput out = new BytesStreamOutput();
+            BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out)
+        ) {
+            task.apply(checksumOut);
+            return checksumOut.getChecksum();
+        }
     }
 
     private void createClusterStateChecksum() {
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 12d10fd908b44..1d7200792442f 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -111,7 +111,7 @@
 import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA;
 import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA;
 import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_METADATA_PREFIX;
-import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteClusterStateConfigured;
 
 /**
  * A Service which provides APIs to upload and download cluster metadata from remote store.
@@ -256,7 +256,7 @@ public RemoteClusterStateService(
         List<IndexMetadataUploadListener> indexMetadataUploadListeners,
         NamedWriteableRegistry namedWriteableRegistry
     ) {
-        assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled";
+        assert isRemoteClusterStateConfigured(settings) : "Remote cluster state is not configured";
         this.nodeId = nodeId;
         this.repositoriesService = repositoriesService;
         this.settings = settings;
@@ -332,7 +332,9 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat
             uploadedMetadataResults,
             previousClusterUUID,
             clusterStateDiffManifest,
-            !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null,
+            !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE)
+                ? new ClusterStateChecksum(clusterState, threadpool)
+                : null,
             false,
             codecVersion
         );
@@ -539,7 +541,9 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(
             uploadedMetadataResults,
             previousManifest.getPreviousClusterUUID(),
             clusterStateDiffManifest,
-            !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null,
+            !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE)
+                ? new ClusterStateChecksum(clusterState, threadpool)
+                : null,
             false,
             previousManifest.getCodecVersion()
         );
@@ -1010,7 +1014,9 @@ public RemoteClusterStateManifestInfo markLastStateAsCommitted(
             uploadedMetadataResults,
             previousManifest.getPreviousClusterUUID(),
             previousManifest.getDiffManifest(),
-            !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null,
+            !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE)
+                ? new ClusterStateChecksum(clusterState, threadpool)
+                : null,
             true,
             previousManifest.getCodecVersion()
         );
@@ -1055,7 +1061,7 @@ public void close() throws IOException {
     }
 
     public void start() {
-        assert isRemoteStoreClusterStateEnabled(settings) == true : "Remote cluster state is not enabled";
+        assert isRemoteClusterStateConfigured(settings) == true : "Remote cluster state is not enabled";
         final String remoteStoreRepo = settings.get(
             Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY
         );
@@ -1631,7 +1637,7 @@ void validateClusterStateFromChecksum(
         String localNodeId,
         boolean isFullStateDownload
     ) {
-        ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState);
+        ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState, threadpool);
         List<String> failedValidation = newClusterStateChecksum.getMismatchEntities(manifest.getClusterStateChecksum());
         if (failedValidation.isEmpty()) {
             return;
@@ -1644,6 +1650,12 @@ void validateClusterStateFromChecksum(
                 failedValidation
             )
         );
+        if (isFullStateDownload) {
+            remoteStateStats.stateFullDownloadValidationFailed();
+        } else {
+            remoteStateStats.stateDiffDownloadValidationFailed();
+        }
+
         if (isFullStateDownload && remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.FAILURE)) {
             throw new IllegalStateException(
                 "Cluster state checksums do not match during full state read. Validation failed for " + failedValidation
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java
new file mode 100644
index 0000000000000..a8f4b33a19c37
--- /dev/null
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway.remote;
+
+import org.opensearch.cluster.coordination.PersistedStateStats;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Download stats for remote state
+ *
+ * @opensearch.internal
+ */
+public class RemoteDownloadStats extends PersistedStateStats {
+    static final String CHECKSUM_VALIDATION_FAILED_COUNT = "checksum_validation_failed_count";
+    private AtomicLong checksumValidationFailedCount = new AtomicLong(0);
+
+    public RemoteDownloadStats(String statsName) {
+        super(statsName);
+        addToExtendedFields(CHECKSUM_VALIDATION_FAILED_COUNT, checksumValidationFailedCount);
+    }
+
+    public void checksumValidationFailedCount() {
+        checksumValidationFailedCount.incrementAndGet();
+    }
+
+    public long getChecksumValidationFailedCount() {
+        return checksumValidationFailedCount.get();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java
index 417ebdafd3ba7..11f26ac8b3ed9 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java
@@ -18,16 +18,16 @@
 public class RemotePersistenceStats {
 
     RemoteUploadStats remoteUploadStats;
-    PersistedStateStats remoteDiffDownloadStats;
-    PersistedStateStats remoteFullDownloadStats;
+    RemoteDownloadStats remoteDiffDownloadStats;
+    RemoteDownloadStats remoteFullDownloadStats;
 
     final String FULL_DOWNLOAD_STATS = "remote_full_download";
     final String DIFF_DOWNLOAD_STATS = "remote_diff_download";
 
     public RemotePersistenceStats() {
         remoteUploadStats = new RemoteUploadStats();
-        remoteDiffDownloadStats = new PersistedStateStats(DIFF_DOWNLOAD_STATS);
-        remoteFullDownloadStats = new PersistedStateStats(FULL_DOWNLOAD_STATS);
+        remoteDiffDownloadStats = new RemoteDownloadStats(DIFF_DOWNLOAD_STATS);
+        remoteFullDownloadStats = new RemoteDownloadStats(FULL_DOWNLOAD_STATS);
     }
 
     public void cleanUpAttemptFailed() {
@@ -90,6 +90,22 @@ public void stateDiffDownloadFailed() {
         remoteDiffDownloadStats.stateFailed();
     }
 
+    public void stateDiffDownloadValidationFailed() {
+        remoteDiffDownloadStats.checksumValidationFailedCount();
+    }
+
+    public void stateFullDownloadValidationFailed() {
+        remoteFullDownloadStats.checksumValidationFailedCount();
+    }
+
+    public long getStateDiffDownloadValidationFailed() {
+        return remoteDiffDownloadStats.getChecksumValidationFailedCount();
+    }
+
+    public long getStateFullDownloadValidationFailed() {
+        return remoteFullDownloadStats.getChecksumValidationFailedCount();
+    }
+
     public PersistedStateStats getUploadStats() {
         return remoteUploadStats;
     }
diff --git a/server/src/main/java/org/opensearch/identity/IdentityService.java b/server/src/main/java/org/opensearch/identity/IdentityService.java
index 03f937180f4ba..33066fae5a80d 100644
--- a/server/src/main/java/org/opensearch/identity/IdentityService.java
+++ b/server/src/main/java/org/opensearch/identity/IdentityService.java
@@ -8,6 +8,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.OpenSearchException;
+import org.opensearch.common.annotation.InternalApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.identity.noop.NoopIdentityPlugin;
 import org.opensearch.identity.tokens.TokenManager;
@@ -22,8 +23,9 @@
 /**
  * Identity and access control for OpenSearch
  *
- * @opensearch.experimental
+ * @opensearch.internal
  * */
+@InternalApi
 public class IdentityService {
     private static final Logger log = LogManager.getLogger(IdentityService.class);
 
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java
index a1d638616f2aa..5e5814528fcd2 100644
--- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java
@@ -34,6 +34,7 @@
 import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode;
 import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType;
 import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator;
+import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator;
 import org.opensearch.index.mapper.DocCountFieldMapper;
 import org.opensearch.index.mapper.FieldMapper;
 import org.opensearch.index.mapper.FieldValueConverter;
@@ -193,7 +194,9 @@ public List<SequentialDocValuesIterator> getMetricReaders(SegmentWriteState stat
                         metricFieldInfo = getFieldInfo(metric.getField(), DocValuesType.SORTED_NUMERIC);
                     }
                     metricReader = new SequentialDocValuesIterator(
-                        fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo)
+                        new SortedNumericStarTreeValuesIterator(
+                            fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo)
+                        )
                     );
                 }
                 metricReaders.add(metricReader);
@@ -228,7 +231,7 @@ public void build(
                 dimensionFieldInfo = getFieldInfo(dimension, DocValuesType.SORTED_NUMERIC);
             }
             dimensionReaders[i] = new SequentialDocValuesIterator(
-                fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo)
+                new SortedNumericStarTreeValuesIterator(fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo))
             );
         }
         Iterator<StarTreeDocument> starTreeDocumentIterator = sortAndAggregateSegmentDocuments(dimensionReaders, metricReaders);
@@ -287,7 +290,7 @@ void appendDocumentsToStarTree(Iterator<StarTreeDocument> starTreeDocumentIterat
         }
     }
 
-    private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDocs) throws IOException {
+    private void serializeStarTree(int numSegmentStarTreeDocuments, int numStarTreeDocs) throws IOException {
         // serialize the star tree data
         long dataFilePointer = dataOut.getFilePointer();
         StarTreeWriter starTreeWriter = new StarTreeWriter();
@@ -299,7 +302,7 @@ private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDo
             starTreeField,
             metricAggregatorInfos,
             numStarTreeNodes,
-            numSegmentStarTreeDocument,
+            numSegmentStarTreeDocuments,
             numStarTreeDocs,
             dataFilePointer,
             totalStarTreeDataLength
@@ -400,22 +403,20 @@ protected StarTreeDocument getStarTreeDocument(
     ) throws IOException {
         Long[] dims = new Long[numDimensions];
         int i = 0;
-        for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) {
-            dimensionDocValueIterator.nextDoc(currentDocId);
-            Long val = dimensionDocValueIterator.value(currentDocId);
+        for (SequentialDocValuesIterator dimensionValueIterator : dimensionReaders) {
+            dimensionValueIterator.nextEntry(currentDocId);
+            Long val = dimensionValueIterator.value(currentDocId);
             dims[i] = val;
             i++;
         }
         i = 0;
         Object[] metrics = new Object[metricReaders.size()];
-        for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) {
-            metricDocValuesIterator.nextDoc(currentDocId);
+        for (SequentialDocValuesIterator metricValuesIterator : metricReaders) {
+            metricValuesIterator.nextEntry(currentDocId);
             // As part of merge, we traverse the star tree doc values
             // The type of data stored in metric fields is different from the
             // actual indexing field they're based on
-            metrics[i] = metricAggregatorInfos.get(i)
-                .getValueAggregators()
-                .toAggregatedValueType(metricDocValuesIterator.value(currentDocId));
+            metrics[i] = metricAggregatorInfos.get(i).getValueAggregators().toAggregatedValueType(metricValuesIterator.value(currentDocId));
             i++;
         }
         return new StarTreeDocument(dims, metrics);
@@ -502,7 +503,7 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIte
         for (int i = 0; i < numDimensions; i++) {
             if (dimensionReaders[i] != null) {
                 try {
-                    dimensionReaders[i].nextDoc(currentDocId);
+                    dimensionReaders[i].nextEntry(currentDocId);
                 } catch (IOException e) {
                     logger.error("unable to iterate to next doc", e);
                     throw new RuntimeException("unable to iterate to next doc", e);
@@ -530,7 +531,7 @@ private Object[] getStarTreeMetricsFromSegment(int currentDocId, List<Sequential
             SequentialDocValuesIterator metricStatReader = metricsReaders.get(i);
             if (metricStatReader != null) {
                 try {
-                    metricStatReader.nextDoc(currentDocId);
+                    metricStatReader.nextEntry(currentDocId);
                 } catch (IOException e) {
                     logger.error("unable to iterate to next doc", e);
                     throw new RuntimeException("unable to iterate to next doc", e);
@@ -672,7 +673,7 @@ private SequentialDocValuesIterator getIteratorForNumericField(
         SequentialDocValuesIterator sequentialDocValuesIterator;
         assert fieldProducerMap.containsKey(fieldInfo.name);
         sequentialDocValuesIterator = new SequentialDocValuesIterator(
-            DocValues.singleton(fieldProducerMap.get(fieldInfo.name).getNumeric(fieldInfo))
+            new SortedNumericStarTreeValuesIterator(DocValues.singleton(fieldProducerMap.get(fieldInfo.name).getNumeric(fieldInfo)))
         );
         return sequentialDocValuesIterator;
     }
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java
index 1613b7c5a3ac0..11cd9e0c94113 100644
--- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java
@@ -153,7 +153,7 @@ Iterator<StarTreeDocument> mergeStarTrees(List<StarTreeValues> starTreeValuesSub
                     .size()];
                 for (int i = 0; i < dimensionsSplitOrder.size(); i++) {
                     String dimension = dimensionsSplitOrder.get(i).getField();
-                    dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension));
+                    dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension));
                 }
                 List<SequentialDocValuesIterator> metricReaders = new ArrayList<>();
                 // get doc id set iterators for metrics
@@ -164,7 +164,7 @@ Iterator<StarTreeDocument> mergeStarTrees(List<StarTreeValues> starTreeValuesSub
                             metric.getField(),
                             metricStat.getTypeName()
                         );
-                        metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName)));
+                        metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName)));
                     }
                 }
                 int currentDocId = 0;
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java
index 1a5c906ad413b..13c6d03c4dc3d 100644
--- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java
@@ -138,7 +138,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List<StarTreeValues> starTreeVal
 
             for (int i = 0; i < dimensionsSplitOrder.size(); i++) {
                 String dimension = dimensionsSplitOrder.get(i).getField();
-                dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension));
+                dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension));
             }
 
             List<SequentialDocValuesIterator> metricReaders = new ArrayList<>();
@@ -150,7 +150,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List<StarTreeValues> starTreeVal
                         metric.getField(),
                         metricStat.getTypeName()
                     );
-                    metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName)));
+                    metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName)));
 
                 }
             }
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java
index 779ed77b0540a..15ed153249243 100644
--- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java
@@ -21,7 +21,6 @@
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -55,11 +54,9 @@ public class StarTreeDocsFileManager extends AbstractDocumentsFileManager implem
     private RandomAccessInput starTreeDocsFileRandomInput;
     private IndexOutput starTreeDocsFileOutput;
     private final Map<String, Integer> fileToEndDocIdMap;
-    private final List<Integer> starTreeDocumentOffsets = new ArrayList<>();
     private int currentFileStartDocId;
     private int numReadableStarTreeDocuments;
     private int starTreeFileCount = -1;
-    private int currBytes = 0;
     private final int fileCountMergeThreshold;
     private int numStarTreeDocs = 0;
 
@@ -98,7 +95,11 @@ IndexOutput createStarTreeDocumentsFileOutput() throws IOException {
     public void writeStarTreeDocument(StarTreeDocument starTreeDocument, boolean isAggregatedDoc) throws IOException {
         assert isAggregatedDoc == true;
         int numBytes = writeStarTreeDocument(starTreeDocument, starTreeDocsFileOutput, true);
-        addStarTreeDocumentOffset(numBytes);
+        if (docSizeInBytes == -1) {
+            docSizeInBytes = numBytes;
+        } else {
+            assert docSizeInBytes == numBytes;
+        }
         numStarTreeDocs++;
     }
 
@@ -106,7 +107,14 @@ public void writeStarTreeDocument(StarTreeDocument starTreeDocument, boolean isA
     public StarTreeDocument readStarTreeDocument(int docId, boolean isAggregatedDoc) throws IOException {
         assert isAggregatedDoc == true;
         ensureDocumentReadable(docId);
-        return readStarTreeDocument(starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId), true);
+        return readStarTreeDocument(starTreeDocsFileRandomInput, getOffset(docId), true);
+    }
+
+    /**
+     * Returns offset for the docId based on the current file start id
+     */
+    private long getOffset(int docId) {
+        return (long) (docId - currentFileStartDocId) * docSizeInBytes;
     }
 
     @Override
@@ -119,19 +127,10 @@ public Long getDimensionValue(int docId, int dimensionId) throws IOException {
     public Long[] readDimensions(int docId) throws IOException {
         ensureDocumentReadable(docId);
         Long[] dims = new Long[starTreeField.getDimensionsOrder().size()];
-        readDimensions(dims, starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId));
+        readDimensions(dims, starTreeDocsFileRandomInput, getOffset(docId));
         return dims;
     }
 
-    private void addStarTreeDocumentOffset(int bytes) {
-        starTreeDocumentOffsets.add(currBytes);
-        currBytes += bytes;
-        if (docSizeInBytes == -1) {
-            docSizeInBytes = bytes;
-        }
-        assert docSizeInBytes == bytes;
-    }
-
     /**
      * Load the correct StarTreeDocuments file based on the docId
      */
@@ -199,7 +198,6 @@ private void loadStarTreeDocumentFile(int docId) throws IOException {
      *    If the operation is only for reading existing documents, a new file is not created.
      */
     private void closeAndMaybeCreateNewFile(boolean shouldCreateFileForAppend, int numStarTreeDocs) throws IOException {
-        currBytes = 0;
         if (starTreeDocsFileOutput != null) {
             fileToEndDocIdMap.put(starTreeDocsFileOutput.getName(), numStarTreeDocs);
             IOUtils.close(starTreeDocsFileOutput);
@@ -232,7 +230,6 @@ private void mergeFiles(int numStarTreeDocs) throws IOException {
             deleteOldFiles();
             fileToEndDocIdMap.clear();
             fileToEndDocIdMap.put(mergedOutput.getName(), numStarTreeDocs);
-            resetStarTreeDocumentOffsets();
         }
     }
 
@@ -259,17 +256,6 @@ private void deleteOldFiles() throws IOException {
         }
     }
 
-    /**
-     * Reset the star tree document offsets based on the merged file
-     */
-    private void resetStarTreeDocumentOffsets() {
-        int curr = 0;
-        for (int i = 0; i < starTreeDocumentOffsets.size(); i++) {
-            starTreeDocumentOffsets.set(i, curr);
-            curr += docSizeInBytes;
-        }
-    }
-
     @Override
     public void close() {
         try {
@@ -288,7 +274,6 @@ public void close() {
                 tmpDirectory.deleteFile(file);
             } catch (IOException ignored) {} // similar to IOUtils.deleteFilesIgnoringExceptions
         }
-        starTreeDocumentOffsets.clear();
         fileToEndDocIdMap.clear();
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java
index a34bbbe9ee738..255ad343cde32 100644
--- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.IndexInput;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.compositeindex.CompositeIndexMetadata;
@@ -25,6 +24,8 @@
 import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata;
 import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory;
 import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode;
+import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator;
+import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -59,14 +60,14 @@ public class StarTreeValues implements CompositeIndexValues {
     private final StarTreeNode root;
 
     /**
-     * A map containing suppliers for DocIdSetIterators for dimensions.
+     * A map containing suppliers for StarTreeValues iterators for dimensions.
      */
-    private final Map<String, Supplier<DocIdSetIterator>> dimensionDocValuesIteratorMap;
+    private final Map<String, Supplier<StarTreeValuesIterator>> dimensionValuesIteratorMap;
 
     /**
-     * A map containing suppliers for DocIdSetIterators for metrics.
+     * A map containing suppliers for StarTreeValues iterators for metrics.
      */
-    private final Map<String, Supplier<DocIdSetIterator>> metricDocValuesIteratorMap;
+    private final Map<String, Supplier<StarTreeValuesIterator>> metricValuesIteratorMap;
 
     /**
      * A map containing attributes associated with the star tree values.
@@ -84,22 +85,22 @@ public class StarTreeValues implements CompositeIndexValues {
      *
      * @param starTreeField                 The StarTreeField object representing the star tree field configuration.
      * @param root                          The root node of the star tree.
-     * @param dimensionDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for dimensions.
-     * @param metricDocValuesIteratorMap    A map containing suppliers for DocIdSetIterators for metrics.
+     * @param dimensionValuesIteratorMap A map containing suppliers for StarTreeValues iterators for dimensions.
+     * @param metricValuesIteratorMap    A map containing suppliers for StarTreeValues iterators for metrics.
      * @param attributes                    A map containing attributes associated with the star tree values.
      */
     public StarTreeValues(
         StarTreeField starTreeField,
         StarTreeNode root,
-        Map<String, Supplier<DocIdSetIterator>> dimensionDocValuesIteratorMap,
-        Map<String, Supplier<DocIdSetIterator>> metricDocValuesIteratorMap,
+        Map<String, Supplier<StarTreeValuesIterator>> dimensionValuesIteratorMap,
+        Map<String, Supplier<StarTreeValuesIterator>> metricValuesIteratorMap,
         Map<String, String> attributes,
         StarTreeMetadata compositeIndexMetadata
     ) {
         this.starTreeField = starTreeField;
         this.root = root;
-        this.dimensionDocValuesIteratorMap = dimensionDocValuesIteratorMap;
-        this.metricDocValuesIteratorMap = metricDocValuesIteratorMap;
+        this.dimensionValuesIteratorMap = dimensionValuesIteratorMap;
+        this.metricValuesIteratorMap = metricValuesIteratorMap;
         this.attributes = attributes;
         this.starTreeMetadata = compositeIndexMetadata;
     }
@@ -146,12 +147,12 @@ public StarTreeValues(
         this.root = StarTreeFactory.createStarTree(compositeIndexDataIn, starTreeMetadata);
 
         // get doc id set iterators for metrics and dimensions
-        dimensionDocValuesIteratorMap = new LinkedHashMap<>();
-        metricDocValuesIteratorMap = new LinkedHashMap<>();
+        dimensionValuesIteratorMap = new LinkedHashMap<>();
+        metricValuesIteratorMap = new LinkedHashMap<>();
 
         // get doc id set iterators for dimensions
         for (String dimension : starTreeMetadata.getDimensionFields()) {
-            dimensionDocValuesIteratorMap.put(dimension, () -> {
+            dimensionValuesIteratorMap.put(dimension, () -> {
                 try {
                     SortedNumericDocValues dimensionSortedNumericDocValues = null;
                     if (readState != null) {
@@ -162,9 +163,9 @@ public StarTreeValues(
                             dimensionSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(dimensionfieldInfo);
                         }
                     }
-                    return getSortedNumericDocValues(dimensionSortedNumericDocValues);
+                    return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(dimensionSortedNumericDocValues));
                 } catch (IOException e) {
-                    throw new RuntimeException("Error loading dimension DocIdSetIterator", e);
+                    throw new RuntimeException("Error loading dimension StarTreeValuesIterator", e);
                 }
             });
         }
@@ -177,7 +178,7 @@ public StarTreeValues(
                     metric.getField(),
                     metricStat.getTypeName()
                 );
-                metricDocValuesIteratorMap.put(metricFullName, () -> {
+                metricValuesIteratorMap.put(metricFullName, () -> {
                     try {
                         SortedNumericDocValues metricSortedNumericDocValues = null;
                         if (readState != null) {
@@ -186,7 +187,7 @@ public StarTreeValues(
                                 metricSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(metricFieldInfo);
                             }
                         }
-                        return getSortedNumericDocValues(metricSortedNumericDocValues);
+                        return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(metricSortedNumericDocValues));
                     } catch (IOException e) {
                         throw new RuntimeException("Error loading metric DocIdSetIterator", e);
                     }
@@ -239,30 +240,30 @@ public Map<String, String> getAttributes() {
     }
 
     /**
-     * Returns the DocIdSetIterator for the specified dimension.
+     * Returns the StarTreeValues iterator for the specified dimension.
      *
      * @param dimension The name of the dimension.
-     * @return The DocIdSetIterator for the specified dimension.
+     * @return The StarTreeValuesIterator for the specified dimension.
      */
-    public DocIdSetIterator getDimensionDocIdSetIterator(String dimension) {
+    public StarTreeValuesIterator getDimensionValuesIterator(String dimension) {
 
-        if (dimensionDocValuesIteratorMap.containsKey(dimension)) {
-            return dimensionDocValuesIteratorMap.get(dimension).get();
+        if (dimensionValuesIteratorMap.containsKey(dimension)) {
+            return dimensionValuesIteratorMap.get(dimension).get();
         }
 
         throw new IllegalArgumentException("dimension [" + dimension + "] does not exist in the segment.");
     }
 
     /**
-     * Returns the DocIdSetIterator for the specified fully qualified metric name.
+     * Returns the StarTreeValues iterator for the specified fully qualified metric name.
      *
      * @param fullyQualifiedMetricName The fully qualified name of the metric.
-     * @return The DocIdSetIterator for the specified fully qualified metric name.
+     * @return The StarTreeValuesIterator for the specified fully qualified metric name.
      */
-    public DocIdSetIterator getMetricDocIdSetIterator(String fullyQualifiedMetricName) {
+    public StarTreeValuesIterator getMetricValuesIterator(String fullyQualifiedMetricName) {
 
-        if (metricDocValuesIteratorMap.containsKey(fullyQualifiedMetricName)) {
-            return metricDocValuesIteratorMap.get(fullyQualifiedMetricName).get();
+        if (metricValuesIteratorMap.containsKey(fullyQualifiedMetricName)) {
+            return metricValuesIteratorMap.get(fullyQualifiedMetricName).get();
         }
 
         throw new IllegalArgumentException("metric [" + fullyQualifiedMetricName + "] does not exist in the segment.");
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java
index 061841d3e140a..9029a451ca4d9 100644
--- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java
@@ -9,15 +9,19 @@
 
 package org.opensearch.index.compositeindex.datacube.startree.utils;
 
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.search.DocIdSetIterator;
 import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator;
+import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator;
 
 import java.io.IOException;
 
 /**
- * Coordinates the reading of documents across multiple DocIdSetIterators.
- * It encapsulates a single DocIdSetIterator and maintains the latest document ID and its associated value.
+ * Coordinates the reading of documents across multiple StarTreeValuesIterator.
+ * It encapsulates a single StarTreeValuesIterator and maintains the latest document ID and its associated value.
+ *
+ * In case of merge , this will be reading the entries of star tree values and in case of flush this will go through
+ * the actual segment documents.
+ *
  * @opensearch.experimental
  */
 @ExperimentalApi
@@ -26,76 +30,56 @@ public class SequentialDocValuesIterator {
     /**
      * The doc id set iterator associated for each field.
      */
-    private final DocIdSetIterator docIdSetIterator;
+    private final StarTreeValuesIterator starTreeValuesIterator;
 
     /**
-     * The value associated with the latest document.
+     * The id of the latest record/entry.
      */
-    private Long docValue;
+    private int entryId = -1;
 
-    /**
-     * The id of the latest document.
-     */
-    private int docId = -1;
-
-    /**
-     * Constructs a new SequentialDocValuesIterator instance with the given DocIdSetIterator.
-     *
-     * @param docIdSetIterator the DocIdSetIterator to be associated with this instance
-     */
-    public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) {
-        this.docIdSetIterator = docIdSetIterator;
-    }
-
-    /**
-     * Returns the id of the latest document.
-     *
-     * @return the id of the latest document
-     */
-    public int getDocId() {
-        return docId;
+    public SequentialDocValuesIterator(StarTreeValuesIterator starTreeValuesIterator) {
+        this.starTreeValuesIterator = starTreeValuesIterator;
     }
 
     /**
-     * Sets the id of the latest document.
+     * Returns the ID of the star tree record/entry or the segment document id
      *
-     * @param docId the ID of the latest document
+     * @return the ID of the star tree record/entry or the segment document id
      */
-    private void setDocId(int docId) {
-        this.docId = docId;
+    public int getEntryId() {
+        return entryId;
     }
 
     /**
-     * Returns the DocIdSetIterator associated with this instance.
+     * Sets the id of the latest entry.
      *
-     * @return the DocIdSetIterator associated with this instance
+     * @param entryId the ID of the star tree record/entry or the segment document id
      */
-    public DocIdSetIterator getDocIdSetIterator() {
-        return docIdSetIterator;
+    private void setEntryId(int entryId) {
+        this.entryId = entryId;
     }
 
-    public int nextDoc(int currentDocId) throws IOException {
+    public int nextEntry(int currentEntryId) throws IOException {
         // if doc id stored is less than or equal to the requested doc id , return the stored doc id
-        if (docId >= currentDocId) {
-            return docId;
+        if (entryId >= currentEntryId) {
+            return entryId;
         }
-        setDocId(this.docIdSetIterator.nextDoc());
-        return docId;
+        setEntryId(this.starTreeValuesIterator.nextEntry());
+        return entryId;
     }
 
-    public Long value(int currentDocId) throws IOException {
-        if (this.getDocIdSetIterator() instanceof SortedNumericDocValues) {
-            SortedNumericDocValues sortedNumericDocValues = (SortedNumericDocValues) this.getDocIdSetIterator();
-            if (currentDocId < 0) {
-                throw new IllegalStateException("invalid doc id to fetch the next value");
+    public Long value(int currentEntryId) throws IOException {
+        if (starTreeValuesIterator instanceof SortedNumericStarTreeValuesIterator) {
+            if (currentEntryId < 0) {
+                throw new IllegalStateException("invalid entry id to fetch the next value");
             }
-            if (currentDocId == DocIdSetIterator.NO_MORE_DOCS) {
-                throw new IllegalStateException("DocValuesIterator is already exhausted");
+            if (currentEntryId == StarTreeValuesIterator.NO_MORE_ENTRIES) {
+                throw new IllegalStateException("StarTreeValuesIterator is already exhausted");
             }
-            if (docId == DocIdSetIterator.NO_MORE_DOCS || docId != currentDocId) {
+            if (entryId == StarTreeValuesIterator.NO_MORE_ENTRIES || entryId != currentEntryId) {
                 return null;
             }
-            return sortedNumericDocValues.nextValue();
+            return ((SortedNumericStarTreeValuesIterator) starTreeValuesIterator).nextValue();
 
         } else {
             throw new IllegalStateException("Unsupported Iterator requested for SequentialDocValuesIterator");
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java
new file mode 100644
index 0000000000000..27afdf1479b4e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java
@@ -0,0 +1,32 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.utils.iterator;
+
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Wrapper iterator class for StarTree index to traverse through SortedNumericDocValues
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class SortedNumericStarTreeValuesIterator extends StarTreeValuesIterator {
+
+    public SortedNumericStarTreeValuesIterator(DocIdSetIterator docIdSetIterator) {
+        super(docIdSetIterator);
+    }
+
+    public long nextValue() throws IOException {
+        return ((SortedNumericDocValues) docIdSetIterator).nextValue();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java
new file mode 100644
index 0000000000000..32866f3e50092
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java
@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.utils.iterator;
+
+import org.apache.lucene.search.DocIdSetIterator;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Wrapper iterator class for StarTree index in place of DocIdSetIterator to read / traverse the docValues formats.
+ * This is needed since star tree values are different from segment documents and number of star tree values
+ * can even exceed segment docs in the worst cases.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public abstract class StarTreeValuesIterator {
+
+    public static final int NO_MORE_ENTRIES = Integer.MAX_VALUE;
+    protected final DocIdSetIterator docIdSetIterator;
+
+    public StarTreeValuesIterator(DocIdSetIterator docIdSetIterator) {
+        this.docIdSetIterator = docIdSetIterator;
+    }
+
+    public int entryId() {
+        return docIdSetIterator.docID();
+    }
+
+    public int nextEntry() throws IOException {
+        return docIdSetIterator.nextDoc();
+    }
+
+    public int advance(int target) throws IOException {
+        return docIdSetIterator.advance(target);
+    }
+
+    public long cost() {
+        return docIdSetIterator.cost();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java
new file mode 100644
index 0000000000000..3c6444a4a5cac
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * This contains classes for StarTreeValues iterators
+ *
+ * @opensearch.experimental
+ */
+package org.opensearch.index.compositeindex.datacube.startree.utils.iterator;
diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java
index bf8f83e1b95df..738efcfafdca1 100644
--- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java
@@ -30,6 +30,7 @@
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.common.lucene.search.AutomatonQueries;
 import org.opensearch.common.xcontent.JsonToStringXContentParser;
+import org.opensearch.core.common.ParsingException;
 import org.opensearch.core.xcontent.DeprecationHandler;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.XContentParser;
@@ -568,31 +569,41 @@ protected void parseCreateField(ParseContext context) throws IOException {
         if (context.externalValueSet()) {
             String value = context.externalValue().toString();
             parseValueAddFields(context, value, fieldType().name());
-        } else if (context.parser().currentToken() != XContentParser.Token.VALUE_NULL) {
-            JsonToStringXContentParser jsonToStringParser = new JsonToStringXContentParser(
-                NamedXContentRegistry.EMPTY,
-                DeprecationHandler.IGNORE_DEPRECATIONS,
-                context.parser(),
-                fieldType().name()
-            );
-            /*
-              JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser
-              It reads the JSON object and parsed to a list of string
-             */
-            XContentParser parser = jsonToStringParser.parseObject();
-
-            XContentParser.Token currentToken;
-            while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                switch (currentToken) {
-                    case FIELD_NAME:
-                        fieldName = parser.currentName();
-                        break;
-                    case VALUE_STRING:
-                        String value = parser.textOrNull();
-                        parseValueAddFields(context, value, fieldName);
-                        break;
+        } else {
+            XContentParser ctxParser = context.parser();
+            if (ctxParser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                if (ctxParser.currentToken() != XContentParser.Token.START_OBJECT) {
+                    throw new ParsingException(
+                        ctxParser.getTokenLocation(),
+                        "[" + this.name() + "] unexpected token [" + ctxParser.currentToken() + "] in flat_object field value"
+                    );
                 }
 
+                JsonToStringXContentParser jsonToStringParser = new JsonToStringXContentParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.IGNORE_DEPRECATIONS,
+                    ctxParser,
+                    fieldType().name()
+                );
+                /*
+                  JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser
+                  It reads the JSON object and parsed to a list of string
+                 */
+                XContentParser parser = jsonToStringParser.parseObject();
+
+                XContentParser.Token currentToken;
+                while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    switch (currentToken) {
+                        case FIELD_NAME:
+                            fieldName = parser.currentName();
+                            break;
+                        case VALUE_STRING:
+                            String value = parser.textOrNull();
+                            parseValueAddFields(context, value, fieldName);
+                            break;
+                    }
+
+                }
             }
         }
 
diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
index dd984373fc9df..b93c82d7a5c7c 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
@@ -171,6 +171,19 @@ public void setIncludeInParent(boolean value) {
         public void setIncludeInRoot(boolean value) {
             includeInRoot = new Explicit<>(value, true);
         }
+
+        public static boolean isParent(ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, MapperService mapperService) {
+            if (parentObjectMapper == null || childObjectMapper == null) {
+                return false;
+            }
+
+            ObjectMapper parent = childObjectMapper.getParentObjectMapper(mapperService);
+            while (parent != null && parent != parentObjectMapper) {
+                childObjectMapper = parent;
+                parent = childObjectMapper.getParentObjectMapper(mapperService);
+            }
+            return parentObjectMapper == parent;
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java
index 0cb416a9b8370..e43e3bda692e7 100644
--- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java
@@ -40,6 +40,7 @@
 import org.apache.lucene.util.automaton.RegExp;
 import org.opensearch.common.lucene.BytesRefs;
 import org.opensearch.common.lucene.Lucene;
+import org.opensearch.common.lucene.search.AutomatonQueries;
 import org.opensearch.common.unit.Fuzziness;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.analysis.IndexAnalyzers;
@@ -464,7 +465,7 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo
                     return existsQuery(context);
                 }
             } else {
-                approximation = matchAllTermsQuery(name(), requiredNGrams);
+                approximation = matchAllTermsQuery(name(), requiredNGrams, caseInsensitive);
             }
             return new WildcardMatchingQuery(name(), approximation, matchPredicate, value, context, this);
         }
@@ -678,7 +679,7 @@ public Query termsQuery(List<?> values, QueryShardContext context) {
             StringBuilder pattern = new StringBuilder();
             for (Object value : values) {
                 String stringVal = BytesRefs.toString(value);
-                builder.add(matchAllTermsQuery(name(), getRequiredNGrams(stringVal)), BooleanClause.Occur.SHOULD);
+                builder.add(matchAllTermsQuery(name(), getRequiredNGrams(stringVal), false), BooleanClause.Occur.SHOULD);
                 expectedValues.add(stringVal);
                 if (pattern.length() > 0) {
                     pattern.append('|');
@@ -688,10 +689,16 @@ public Query termsQuery(List<?> values, QueryShardContext context) {
             return new WildcardMatchingQuery(name(), builder.build(), expectedValues::contains, pattern.toString(), context, this);
         }
 
-        private static BooleanQuery matchAllTermsQuery(String fieldName, Set<String> terms) {
+        private static BooleanQuery matchAllTermsQuery(String fieldName, Set<String> terms, boolean caseInsensitive) {
             BooleanQuery.Builder matchAllTermsBuilder = new BooleanQuery.Builder();
+            Query query;
             for (String term : terms) {
-                matchAllTermsBuilder.add(new TermQuery(new Term(fieldName, term)), BooleanClause.Occur.FILTER);
+                if (caseInsensitive) {
+                    query = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, term));
+                } else {
+                    query = new TermQuery(new Term(fieldName, term));
+                }
+                matchAllTermsBuilder.add(query, BooleanClause.Occur.FILTER);
             }
             return matchAllTermsBuilder.build();
         }
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java
index 5878dff03acc2..2a76a5b966884 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java
@@ -51,8 +51,8 @@
 import static org.opensearch.index.remote.RemoteIndexPath.SEGMENT_PATH;
 import static org.opensearch.index.remote.RemoteIndexPath.TRANSLOG_PATH;
 import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStorePathStrategy;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteClusterStateConfigured;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent;
-import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
 
 /**
  * Uploads the remote store path for all possible combinations of {@link org.opensearch.index.remote.RemoteStoreEnums.DataCategory}
@@ -235,7 +235,7 @@ private Repository validateAndGetRepository(String repoSetting) {
     }
 
     public void start() {
-        assert isRemoteStoreClusterStateEnabled(settings) == true : "Remote cluster state is not enabled";
+        assert isRemoteClusterStateConfigured(settings) == true : "Remote cluster state is not configured";
         if (isRemoteDataAttributePresent == false) {
             // If remote store data attributes are not present than we skip this.
             return;
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java
index edcfe17d31fb8..32a1ca0e5d5ab 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java
@@ -477,6 +477,7 @@ private static Set<String> getPinnedTimestampLockedFiles(
         for (Long pinnedTimestamp : pinnedTimestampSet) {
             String cachedFile = metadataFilePinnedTimestampMap.get(pinnedTimestamp);
             if (cachedFile != null) {
+                assert metadataFiles.contains(cachedFile) : "Metadata files should contain [" + cachedFile + "]";
                 implicitLockedFiles.add(cachedFile);
             } else {
                 newPinnedTimestamps.add(pinnedTimestamp);
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
index 72bf07d4b03b2..5be516166803e 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
@@ -854,7 +854,7 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException
         }
 
         // Check last fetch status of pinned timestamps. If stale, return.
-        if (RemoteStoreUtils.isPinnedTimestampStateStale()) {
+        if (lastNMetadataFilesToKeep != 0 && RemoteStoreUtils.isPinnedTimestampStateStale()) {
             logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale");
             return;
         }
@@ -994,7 +994,8 @@ public static void remoteDirectoryCleanup(
         String remoteStoreRepoForIndex,
         String indexUUID,
         ShardId shardId,
-        RemoteStorePathStrategy pathStrategy
+        RemoteStorePathStrategy pathStrategy,
+        boolean forceClean
     ) {
         try {
             RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = (RemoteSegmentStoreDirectory) remoteDirectoryFactory.newDirectory(
@@ -1003,8 +1004,12 @@ public static void remoteDirectoryCleanup(
                 shardId,
                 pathStrategy
             );
-            remoteSegmentStoreDirectory.deleteStaleSegments(0);
-            remoteSegmentStoreDirectory.deleteIfEmpty();
+            if (forceClean) {
+                remoteSegmentStoreDirectory.delete();
+            } else {
+                remoteSegmentStoreDirectory.deleteStaleSegments(0);
+                remoteSegmentStoreDirectory.deleteIfEmpty();
+            }
         } catch (Exception e) {
             staticLogger.error("Exception occurred while deleting directory", e);
         }
@@ -1023,7 +1028,10 @@ private boolean deleteIfEmpty() throws IOException {
             logger.info("Remote directory still has files, not deleting the path");
             return false;
         }
+        return delete();
+    }
 
+    private boolean delete() {
         try {
             remoteDataDirectory.delete();
             remoteMetadataDirectory.delete();
diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java
index 27d34ec0d05af..3ccacde22bbfc 100644
--- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java
+++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java
@@ -34,6 +34,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.BooleanSupplier;
 import java.util.function.LongConsumer;
@@ -61,6 +62,7 @@ public class RemoteFsTimestampAwareTranslog extends RemoteFsTranslog {
     private final Map<String, Tuple<Long, Long>> oldFormatMetadataFileGenerationMap;
     private final Map<String, Tuple<Long, Long>> oldFormatMetadataFilePrimaryTermMap;
     private final AtomicLong minPrimaryTermInRemote = new AtomicLong(Long.MAX_VALUE);
+    private final AtomicBoolean triggerTrimOnMinRemoteGenReferencedChange = new AtomicBoolean(false);
 
     public RemoteFsTimestampAwareTranslog(
         TranslogConfig config,
@@ -105,6 +107,11 @@ protected void onDelete() {
         }
     }
 
+    @Override
+    protected void onMinRemoteGenReferencedChange() {
+        triggerTrimOnMinRemoteGenReferencedChange.set(true);
+    }
+
     @Override
     public void trimUnreferencedReaders() throws IOException {
         trimUnreferencedReaders(false, true);
@@ -114,7 +121,7 @@ public void trimUnreferencedReaders() throws IOException {
     protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal) throws IOException {
         if (trimLocal) {
             // clean up local translog files and updates readers
-            super.trimUnreferencedReaders();
+            super.trimUnreferencedReaders(true);
         }
 
         // Update file tracker to reflect local translog state
@@ -135,14 +142,22 @@ protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal)
 
         // This is to ensure that after the permits are acquired during primary relocation, there are no further modification on remote
         // store.
-        if (startedPrimarySupplier.getAsBoolean() == false || pauseSync.get()) {
+        if (indexDeleted == false && (startedPrimarySupplier.getAsBoolean() == false || pauseSync.get())) {
             return;
         }
 
         // This is to fail fast and avoid listing md files un-necessarily.
         if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) {
-            logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale");
+            logger.warn("Skipping remote translog garbage collection as last fetch of pinned timestamp is stale");
+            return;
+        }
+
+        // This code block ensures parity with RemoteFsTranslog. Without this, we will end up making list translog metadata
+        // call in each invocation of trimUnreferencedReaders
+        if (indexDeleted == false && triggerTrimOnMinRemoteGenReferencedChange.get() == false) {
             return;
+        } else if (triggerTrimOnMinRemoteGenReferencedChange.get()) {
+            triggerTrimOnMinRemoteGenReferencedChange.set(false);
         }
 
         // Since remote generation deletion is async, this ensures that only one generation deletion happens at a time.
@@ -158,7 +173,7 @@ public void onResponse(List<BlobMetadata> blobMetadata) {
                 List<String> metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList());
 
                 try {
-                    if (metadataFiles.size() <= 1) {
+                    if (indexDeleted == false && metadataFiles.size() <= 1) {
                         logger.debug("No stale translog metadata files found");
                         remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS);
                         return;
@@ -166,16 +181,12 @@ public void onResponse(List<BlobMetadata> blobMetadata) {
 
                     // Check last fetch status of pinned timestamps. If stale, return.
                     if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) {
-                        logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale");
+                        logger.warn("Skipping remote translog garbage collection as last fetch of pinned timestamp is stale");
                         remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS);
                         return;
                     }
 
-                    List<String> metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(
-                        metadataFiles,
-                        metadataFilePinnedTimestampMap,
-                        logger
-                    );
+                    List<String> metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, indexDeleted);
 
                     // If index is not deleted, make sure to keep latest metadata file
                     if (indexDeleted == false) {
@@ -194,10 +205,11 @@ public void onResponse(List<BlobMetadata> blobMetadata) {
                     metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted);
 
                     logger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted);
+
                     Set<Long> generationsToBeDeleted = getGenerationsToBeDeleted(
                         metadataFilesNotToBeDeleted,
                         metadataFilesToBeDeleted,
-                        indexDeleted
+                        indexDeleted ? Long.MAX_VALUE : getMinGenerationToKeepInRemote()
                     );
 
                     logger.debug(() -> "generationsToBeDeleted = " + generationsToBeDeleted);
@@ -208,7 +220,11 @@ public void onResponse(List<BlobMetadata> blobMetadata) {
                             generationsToBeDeleted,
                             remoteGenerationDeletionPermits::release
                         );
+                    } else {
+                        remoteGenerationDeletionPermits.release();
+                    }
 
+                    if (metadataFilesToBeDeleted.isEmpty() == false) {
                         // Delete stale metadata files
                         translogTransferManager.deleteMetadataFilesAsync(
                             metadataFilesToBeDeleted,
@@ -217,11 +233,11 @@ public void onResponse(List<BlobMetadata> blobMetadata) {
 
                         // Update cache to keep only those metadata files that are not getting deleted
                         oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted);
-
+                        oldFormatMetadataFilePrimaryTermMap.keySet().retainAll(metadataFilesNotToBeDeleted);
                         // Delete stale primary terms
                         deleteStaleRemotePrimaryTerms(metadataFilesNotToBeDeleted);
                     } else {
-                        remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS);
+                        remoteGenerationDeletionPermits.release();
                     }
                 } catch (Exception e) {
                     remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS);
@@ -237,18 +253,16 @@ public void onFailure(Exception e) {
         translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener);
     }
 
+    private long getMinGenerationToKeepInRemote() {
+        return minRemoteGenReferenced - indexSettings().getRemoteTranslogExtraKeep();
+    }
+
     // Visible for testing
     protected Set<Long> getGenerationsToBeDeleted(
         List<String> metadataFilesNotToBeDeleted,
         List<String> metadataFilesToBeDeleted,
-        boolean indexDeleted
+        long minGenerationToKeepInRemote
     ) throws IOException {
-        long maxGenerationToBeDeleted = Long.MAX_VALUE;
-
-        if (indexDeleted == false) {
-            maxGenerationToBeDeleted = minRemoteGenReferenced - 1 - indexSettings().getRemoteTranslogExtraKeep();
-        }
-
         Set<Long> generationsFromMetadataFilesToBeDeleted = new HashSet<>();
         for (String mdFile : metadataFilesToBeDeleted) {
             Tuple<Long, Long> minMaxGen = getMinMaxTranslogGenerationFromMetadataFile(mdFile, translogTransferManager);
@@ -262,21 +276,31 @@ protected Set<Long> getGenerationsToBeDeleted(
         Set<Long> generationsToBeDeleted = new HashSet<>();
         for (long generation : generationsFromMetadataFilesToBeDeleted) {
             // Check if the generation is not referred by metadata file matching pinned timestamps
-            if (generation <= maxGenerationToBeDeleted && isGenerationPinned(generation, pinnedGenerations) == false) {
+            // The check with minGenerationToKeep is redundant but kept as to make sure we don't delete generations
+            // that are not persisted in remote segment store yet.
+            if (generation < minGenerationToKeepInRemote && isGenerationPinned(generation, pinnedGenerations) == false) {
                 generationsToBeDeleted.add(generation);
             }
         }
         return generationsToBeDeleted;
     }
 
-    protected List<String> getMetadataFilesToBeDeleted(List<String> metadataFiles) {
-        return getMetadataFilesToBeDeleted(metadataFiles, metadataFilePinnedTimestampMap, logger);
+    protected List<String> getMetadataFilesToBeDeleted(List<String> metadataFiles, boolean indexDeleted) {
+        return getMetadataFilesToBeDeleted(
+            metadataFiles,
+            metadataFilePinnedTimestampMap,
+            getMinGenerationToKeepInRemote(),
+            indexDeleted,
+            logger
+        );
     }
 
     // Visible for testing
     protected static List<String> getMetadataFilesToBeDeleted(
         List<String> metadataFiles,
         Map<Long, String> metadataFilePinnedTimestampMap,
+        long minGenerationToKeepInRemote,
+        boolean indexDeleted,
         Logger logger
     ) {
         Tuple<Long, Set<Long>> pinnedTimestampsState = RemoteStorePinnedTimestampService.getPinnedTimestamps();
@@ -312,6 +336,22 @@ protected static List<String> getMetadataFilesToBeDeleted(
             metadataFilesToBeDeleted.size()
         );
 
+        if (indexDeleted == false) {
+            // Filter out metadata files based on minGenerationToKeep
+            List<String> metadataFilesContainingMinGenerationToKeep = metadataFilesToBeDeleted.stream().filter(md -> {
+                long maxGeneration = TranslogTransferMetadata.getMaxGenerationFromFileName(md);
+                return maxGeneration == -1 || maxGeneration >= minGenerationToKeepInRemote;
+            }).collect(Collectors.toList());
+            metadataFilesToBeDeleted.removeAll(metadataFilesContainingMinGenerationToKeep);
+
+            logger.trace(
+                "metadataFilesContainingMinGenerationToKeep.size = {}, metadataFilesToBeDeleted based on minGenerationToKeep filtering = {}, minGenerationToKeep = {}",
+                metadataFilesContainingMinGenerationToKeep.size(),
+                metadataFilesToBeDeleted.size(),
+                minGenerationToKeepInRemote
+            );
+        }
+
         return metadataFilesToBeDeleted;
     }
 
@@ -369,9 +409,9 @@ protected Tuple<Long, Long> getMinMaxTranslogGenerationFromMetadataFile(
         }
     }
 
-    private void deleteStaleRemotePrimaryTerms(List<String> metadataFiles) {
+    private void deleteStaleRemotePrimaryTerms(List<String> metadataFilesNotToBeDeleted) {
         deleteStaleRemotePrimaryTerms(
-            metadataFiles,
+            metadataFilesNotToBeDeleted,
             translogTransferManager,
             oldFormatMetadataFilePrimaryTermMap,
             minPrimaryTermInRemote,
@@ -386,7 +426,7 @@ private void deleteStaleRemotePrimaryTerms(List<String> metadataFiles) {
      * This will also delete all stale translog metadata files from remote except the latest basis the metadata file comparator.
      */
     protected static void deleteStaleRemotePrimaryTerms(
-        List<String> metadataFiles,
+        List<String> metadataFilesNotToBeDeleted,
         TranslogTransferManager translogTransferManager,
         Map<String, Tuple<Long, Long>> oldFormatMetadataFilePrimaryTermMap,
         AtomicLong minPrimaryTermInRemoteAtomicLong,
@@ -395,15 +435,15 @@ protected static void deleteStaleRemotePrimaryTerms(
         // The deletion of older translog files in remote store is on best-effort basis, there is a possibility that there
         // are older files that are no longer needed and should be cleaned up. In here, we delete all files that are part
         // of older primary term.
-        if (metadataFiles.isEmpty()) {
+        if (metadataFilesNotToBeDeleted.isEmpty()) {
             logger.trace("No metadata is uploaded yet, returning from deleteStaleRemotePrimaryTerms");
             return;
         }
-        Optional<Long> minPrimaryTermFromMetadataFiles = metadataFiles.stream().map(file -> {
+        Optional<Long> minPrimaryTermFromMetadataFiles = metadataFilesNotToBeDeleted.stream().map(file -> {
             try {
                 return getMinMaxPrimaryTermFromMetadataFile(file, translogTransferManager, oldFormatMetadataFilePrimaryTermMap).v1();
             } catch (IOException e) {
-                return Long.MAX_VALUE;
+                return Long.MIN_VALUE;
             }
         }).min(Long::compareTo);
         // First we delete all stale primary terms folders from remote store
@@ -420,7 +460,7 @@ protected static void deleteStaleRemotePrimaryTerms(
         }
     }
 
-    private static Long getMinPrimaryTermInRemote(
+    protected static Long getMinPrimaryTermInRemote(
         AtomicLong minPrimaryTermInRemote,
         TranslogTransferManager translogTransferManager,
         Logger logger
@@ -472,50 +512,60 @@ protected static Tuple<Long, Long> getMinMaxPrimaryTermFromMetadataFile(
         }
     }
 
-    public static void cleanup(TranslogTransferManager translogTransferManager) throws IOException {
-        ActionListener<List<BlobMetadata>> listMetadataFilesListener = new ActionListener<>() {
-            @Override
-            public void onResponse(List<BlobMetadata> blobMetadata) {
-                List<String> metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList());
+    public static void cleanupOfDeletedIndex(TranslogTransferManager translogTransferManager, boolean forceClean) throws IOException {
+        if (forceClean) {
+            translogTransferManager.delete();
+        } else {
+            ActionListener<List<BlobMetadata>> listMetadataFilesListener = new ActionListener<>() {
+                @Override
+                public void onResponse(List<BlobMetadata> blobMetadata) {
+                    List<String> metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList());
+
+                    try {
+                        if (metadataFiles.isEmpty()) {
+                            staticLogger.debug("No stale translog metadata files found");
+                            return;
+                        }
+                        List<String> metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(
+                            metadataFiles,
+                            new HashMap<>(),
+                            Long.MAX_VALUE,
+                            true,  // This method gets called when the index is no longer present
+                            staticLogger
+                        );
+                        if (metadataFilesToBeDeleted.isEmpty()) {
+                            staticLogger.debug("No metadata files to delete");
+                            return;
+                        }
+                        staticLogger.debug(() -> "metadataFilesToBeDeleted = " + metadataFilesToBeDeleted);
 
-                try {
-                    if (metadataFiles.isEmpty()) {
-                        staticLogger.debug("No stale translog metadata files found");
-                        return;
-                    }
-                    List<String> metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, new HashMap<>(), staticLogger);
-                    if (metadataFilesToBeDeleted.isEmpty()) {
-                        staticLogger.debug("No metadata files to delete");
-                        return;
-                    }
-                    staticLogger.debug(() -> "metadataFilesToBeDeleted = " + metadataFilesToBeDeleted);
+                        // For all the files that we are keeping, fetch min and max generations
+                        List<String> metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles);
+                        metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted);
+                        staticLogger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted);
 
-                    // For all the files that we are keeping, fetch min and max generations
-                    List<String> metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles);
-                    metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted);
-                    staticLogger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted);
+                        // Delete stale metadata files
+                        translogTransferManager.deleteMetadataFilesAsync(metadataFilesToBeDeleted, () -> {});
 
-                    // Delete stale metadata files
-                    translogTransferManager.deleteMetadataFilesAsync(metadataFilesToBeDeleted, () -> {});
+                        // Delete stale primary terms
+                        deleteStaleRemotePrimaryTerms(
+                            metadataFilesNotToBeDeleted,
+                            translogTransferManager,
+                            new HashMap<>(),
+                            new AtomicLong(Long.MAX_VALUE),
+                            staticLogger
+                        );
+                    } catch (Exception e) {
+                        staticLogger.error("Exception while cleaning up metadata and primary terms", e);
+                    }
+                }
 
-                    // Delete stale primary terms
-                    deleteStaleRemotePrimaryTerms(
-                        metadataFilesNotToBeDeleted,
-                        translogTransferManager,
-                        new HashMap<>(),
-                        new AtomicLong(Long.MAX_VALUE),
-                        staticLogger
-                    );
-                } catch (Exception e) {
+                @Override
+                public void onFailure(Exception e) {
                     staticLogger.error("Exception while cleaning up metadata and primary terms", e);
                 }
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                staticLogger.error("Exception while cleaning up metadata and primary terms", e);
-            }
-        };
-        translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener);
+            };
+            translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener);
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
index 812852d107682..f5a9ed8ed9362 100644
--- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
+++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
@@ -78,6 +78,9 @@ public class RemoteFsTranslog extends Translog {
     // min generation referred by last uploaded translog
     protected volatile long minRemoteGenReferenced;
 
+    // the max global checkpoint that has been synced
+    protected volatile long globalCheckpointSynced;
+
     // clean up translog folder uploaded by previous primaries once
     protected final SetOnce<Boolean> olderPrimaryCleaned = new SetOnce<>();
 
@@ -437,9 +440,10 @@ private boolean upload(long primaryTerm, long generation, long maxSeqNo) throws
                 config.getNodeId()
             ).build()
         ) {
+            Checkpoint checkpoint = current.getLastSyncedCheckpoint();
             return translogTransferManager.transferSnapshot(
                 transferSnapshotProvider,
-                new RemoteFsTranslogTransferListener(generation, primaryTerm, maxSeqNo)
+                new RemoteFsTranslogTransferListener(generation, primaryTerm, maxSeqNo, checkpoint.globalCheckpoint)
             );
         } finally {
             syncPermit.release(SYNC_PERMIT);
@@ -474,7 +478,10 @@ public void sync() throws IOException {
     public boolean syncNeeded() {
         try (ReleasableLock lock = readLock.acquire()) {
             return current.syncNeeded()
-                || (maxRemoteTranslogGenerationUploaded + 1 < this.currentFileGeneration() && current.totalOperations() == 0);
+                || (maxRemoteTranslogGenerationUploaded + 1 < this.currentFileGeneration() && current.totalOperations() == 0)
+                // The below condition on GCP exists to handle global checkpoint updates during close index.
+                // Refer issue - https://github.com/opensearch-project/OpenSearch/issues/15989
+                || (current.getLastSyncedCheckpoint().globalCheckpoint > globalCheckpointSynced);
         }
     }
 
@@ -549,9 +556,17 @@ protected Releasable drainSync() {
 
     @Override
     public void trimUnreferencedReaders() throws IOException {
+        trimUnreferencedReaders(false);
+    }
+
+    protected void trimUnreferencedReaders(boolean onlyTrimLocal) throws IOException {
         // clean up local translog files and updates readers
         super.trimUnreferencedReaders();
 
+        if (onlyTrimLocal) {
+            return;
+        }
+
         // This is to ensure that after the permits are acquired during primary relocation, there are no further modification on remote
         // store.
         if (startedPrimarySupplier.getAsBoolean() == false || pauseSync.get()) {
@@ -674,21 +689,34 @@ private class RemoteFsTranslogTransferListener implements TranslogTransferListen
 
         private final long maxSeqNo;
 
-        RemoteFsTranslogTransferListener(long generation, long primaryTerm, long maxSeqNo) {
+        private final long globalCheckpoint;
+
+        RemoteFsTranslogTransferListener(long generation, long primaryTerm, long maxSeqNo, long globalCheckpoint) {
             this.generation = generation;
             this.primaryTerm = primaryTerm;
             this.maxSeqNo = maxSeqNo;
+            this.globalCheckpoint = globalCheckpoint;
         }
 
         @Override
         public void onUploadComplete(TransferSnapshot transferSnapshot) throws IOException {
             maxRemoteTranslogGenerationUploaded = generation;
+            long previousMinRemoteGenReferenced = minRemoteGenReferenced;
             minRemoteGenReferenced = getMinFileGeneration();
+            // Update the global checkpoint only if the supplied global checkpoint is greater than it
+            // When a new writer is created the
+            if (globalCheckpoint > globalCheckpointSynced) {
+                globalCheckpointSynced = globalCheckpoint;
+            }
+            if (previousMinRemoteGenReferenced != minRemoteGenReferenced) {
+                onMinRemoteGenReferencedChange();
+            }
             logger.debug(
-                "Successfully uploaded translog for primary term = {}, generation = {}, maxSeqNo = {}",
+                "Successfully uploaded translog for primary term = {}, generation = {}, maxSeqNo = {}, minRemoteGenReferenced = {}",
                 primaryTerm,
                 generation,
-                maxSeqNo
+                maxSeqNo,
+                minRemoteGenReferenced
             );
         }
 
@@ -702,6 +730,10 @@ public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) thro
         }
     }
 
+    protected void onMinRemoteGenReferencedChange() {
+
+    }
+
     @Override
     public long getMinUnreferencedSeqNoInSegments(long minUnrefCheckpointInLastCommit) {
         return minSeqNoToKeep;
diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java
index 3b8885055e8f7..7fe3305545085 100644
--- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java
@@ -170,6 +170,16 @@ public static Tuple<Long, Long> getMinMaxTranslogGenerationFromFilename(String f
         }
     }
 
+    public static long getMaxGenerationFromFileName(String filename) {
+        String[] tokens = filename.split(METADATA_SEPARATOR);
+        try {
+            return RemoteStoreUtils.invertLong(tokens[2]);
+        } catch (Exception e) {
+            logger.error(() -> new ParameterizedMessage("Exception while getting max generation from: {}", filename), e);
+            return -1;
+        }
+    }
+
     public static Tuple<Long, Long> getMinMaxPrimaryTermFromFilename(String filename) {
         String[] tokens = filename.split(METADATA_SEPARATOR);
         if (tokens.length < 7) {
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index a8d4ebcf23dab..58509aad57eb9 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -41,6 +41,7 @@
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.OpenSearchTimeoutException;
 import org.opensearch.Version;
+import org.opensearch.accesscontrol.resources.ResourceService;
 import org.opensearch.action.ActionModule;
 import org.opensearch.action.ActionModule.DynamicActionRegistry;
 import org.opensearch.action.ActionType;
@@ -213,6 +214,8 @@
 import org.opensearch.plugins.Plugin;
 import org.opensearch.plugins.PluginsService;
 import org.opensearch.plugins.RepositoryPlugin;
+import org.opensearch.plugins.ResourceAccessControlPlugin;
+import org.opensearch.plugins.ResourcePlugin;
 import org.opensearch.plugins.ScriptPlugin;
 import org.opensearch.plugins.SearchPipelinePlugin;
 import org.opensearch.plugins.SearchPlugin;
@@ -311,9 +314,9 @@
 import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath;
 import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY;
 import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteClusterStateConfigured;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreAttributePresent;
-import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
 
 /**
  * A node represent a node within a cluster ({@code cluster.name}). The {@link #client()} can be used
@@ -526,11 +529,7 @@ protected Node(
             FeatureFlags.initializeFeatureFlags(settings);
 
             final List<IdentityPlugin> identityPlugins = new ArrayList<>();
-            if (FeatureFlags.isEnabled(FeatureFlags.IDENTITY)) {
-                // If identity is enabled load plugins implementing the extension point
-                logger.info("Identity on so found plugins implementing: " + pluginsService.filterPlugins(IdentityPlugin.class).toString());
-                identityPlugins.addAll(pluginsService.filterPlugins(IdentityPlugin.class));
-            }
+            identityPlugins.addAll(pluginsService.filterPlugins(IdentityPlugin.class));
 
             final Set<DiscoveryNodeRole> additionalRoles = pluginsService.filterPlugins(Plugin.class)
                 .stream()
@@ -796,7 +795,7 @@ protected Node(
             final RemoteClusterStateService remoteClusterStateService;
             final RemoteClusterStateCleanupManager remoteClusterStateCleanupManager;
             final RemoteIndexPathUploader remoteIndexPathUploader;
-            if (isRemoteStoreClusterStateEnabled(settings)) {
+            if (isRemoteClusterStateConfigured(settings)) {
                 remoteIndexPathUploader = new RemoteIndexPathUploader(
                     threadPool,
                     settings,
@@ -1066,6 +1065,12 @@ protected Node(
             );
             modules.add(actionModule);
 
+            final List<ResourceAccessControlPlugin> resourceAccessControlPlugins = pluginsService.filterPlugins(
+                ResourceAccessControlPlugin.class
+            );
+            final List<ResourcePlugin> resourcePlugins = pluginsService.filterPlugins(ResourcePlugin.class);
+            ResourceService resourceService = new ResourceService(resourceAccessControlPlugins, resourcePlugins);
+
             final RestController restController = actionModule.getRestController();
 
             final NodeResourceUsageTracker nodeResourceUsageTracker = new NodeResourceUsageTracker(
@@ -1466,6 +1471,7 @@ protected Node(
                 b.bind(ResourceUsageCollectorService.class).toInstance(resourceUsageCollectorService);
                 b.bind(SystemIndices.class).toInstance(systemIndices);
                 b.bind(IdentityService.class).toInstance(identityService);
+                b.bind(ResourceService.class).toInstance(resourceService);
                 b.bind(Tracer.class).toInstance(tracer);
                 b.bind(SearchRequestStats.class).toInstance(searchRequestStats);
                 b.bind(SearchRequestSlowLog.class).toInstance(searchRequestSlowLog);
@@ -1602,6 +1608,7 @@ public Node start() throws NodeValidationException {
 
         injector.getInstance(GatewayService.class).start();
         Discovery discovery = injector.getInstance(Discovery.class);
+        discovery.setNodeConnectionsService(nodeConnectionsService);
         clusterService.getClusterManagerService().setClusterStatePublisher(discovery::publish);
 
         // Start the transport service now so the publish address will be added to the local disco node in ClusterService
diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java
index 1eb38ea63ad5a..9671fda14375d 100644
--- a/server/src/main/java/org/opensearch/node/NodeService.java
+++ b/server/src/main/java/org/opensearch/node/NodeService.java
@@ -54,6 +54,7 @@
 import org.opensearch.indices.IndicesService;
 import org.opensearch.ingest.IngestService;
 import org.opensearch.monitor.MonitorService;
+import org.opensearch.node.remotestore.RemoteStoreNodeStats;
 import org.opensearch.plugins.PluginsService;
 import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlService;
 import org.opensearch.repositories.RepositoriesService;
@@ -241,7 +242,8 @@ public NodeStats stats(
         boolean segmentReplicationTrackerStats,
         boolean repositoriesStats,
         boolean admissionControl,
-        boolean cacheService
+        boolean cacheService,
+        boolean remoteStoreNodeStats
     ) {
         // for indices stats we want to include previous allocated shards stats as well (it will
         // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats)
@@ -274,7 +276,8 @@ public NodeStats stats(
             segmentReplicationTrackerStats ? this.segmentReplicationStatsTracker.getTotalRejectionStats() : null,
             repositoriesStats ? this.repositoriesService.getRepositoriesStats() : null,
             admissionControl ? this.admissionControlService.stats() : null,
-            cacheService ? this.cacheService.stats(indices) : null
+            cacheService ? this.cacheService.stats(indices) : null,
+            remoteStoreNodeStats ? new RemoteStoreNodeStats() : null
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java
index 55971398634c5..d6a58f8e1d471 100644
--- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java
+++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java
@@ -180,7 +180,7 @@ public static boolean isRemoteDataAttributePresent(Settings settings) {
             || settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false;
     }
 
-    public static boolean isRemoteClusterStateAttributePresent(Settings settings) {
+    public static boolean isRemoteClusterStateConfigured(Settings settings) {
         return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)
             .isEmpty() == false;
     }
@@ -194,8 +194,7 @@ public static String getRemoteStoreTranslogRepo(Settings settings) {
     }
 
     public static boolean isRemoteStoreClusterStateEnabled(Settings settings) {
-        return RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings)
-            && isRemoteClusterStateAttributePresent(settings);
+        return RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) && isRemoteClusterStateConfigured(settings);
     }
 
     private static boolean isRemoteRoutingTableAttributePresent(Settings settings) {
diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java
new file mode 100644
index 0000000000000..8da8a17e21839
--- /dev/null
+++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java
@@ -0,0 +1,79 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node.remotestore;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Node level remote store stats
+ * @opensearch.internal
+ */
+public class RemoteStoreNodeStats implements Writeable, ToXContentFragment {
+
+    public static final String STATS_NAME = "remote_store";
+    public static final String LAST_SUCCESSFUL_FETCH_OF_PINNED_TIMESTAMPS = "last_successful_fetch_of_pinned_timestamps";
+
+    /**
+     * Time stamp for the last successful fetch of pinned timestamps by the {@linkplain  RemoteStorePinnedTimestampService}
+     */
+    private final long lastSuccessfulFetchOfPinnedTimestamps;
+
+    public RemoteStoreNodeStats() {
+        this.lastSuccessfulFetchOfPinnedTimestamps = RemoteStorePinnedTimestampService.getPinnedTimestamps().v1();
+    }
+
+    public long getLastSuccessfulFetchOfPinnedTimestamps() {
+        return this.lastSuccessfulFetchOfPinnedTimestamps;
+    }
+
+    public RemoteStoreNodeStats(StreamInput in) throws IOException {
+        this.lastSuccessfulFetchOfPinnedTimestamps = in.readLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeLong(this.lastSuccessfulFetchOfPinnedTimestamps);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(STATS_NAME);
+        builder.field(LAST_SUCCESSFUL_FETCH_OF_PINNED_TIMESTAMPS, this.lastSuccessfulFetchOfPinnedTimestamps);
+        return builder.endObject();
+    }
+
+    @Override
+    public String toString() {
+        return "RemoteStoreNodeStats{ lastSuccessfulFetchOfPinnedTimestamps=" + lastSuccessfulFetchOfPinnedTimestamps + "}";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == null) {
+            return false;
+        }
+        if (o.getClass() != RemoteStoreNodeStats.class) {
+            return false;
+        }
+        RemoteStoreNodeStats other = (RemoteStoreNodeStats) o;
+        return this.lastSuccessfulFetchOfPinnedTimestamps == other.lastSuccessfulFetchOfPinnedTimestamps;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(lastSuccessfulFetchOfPinnedTimestamps);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java
index 71133615ed056..a3382d8568ec5 100644
--- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java
+++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java
@@ -30,6 +30,9 @@
 import java.io.ByteArrayInputStream;
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -49,6 +52,7 @@
 public class RemoteStorePinnedTimestampService implements Closeable {
     private static final Logger logger = LogManager.getLogger(RemoteStorePinnedTimestampService.class);
     private static Tuple<Long, Set<Long>> pinnedTimestampsSet = new Tuple<>(-1L, Set.of());
+    private static Map<String, List<Long>> pinnedEntityToTimestampsMap = new HashMap<>();
     public static final String PINNED_TIMESTAMPS_PATH_TOKEN = "pinned_timestamps";
     public static final String PINNED_TIMESTAMPS_FILENAME_SEPARATOR = "__";
 
@@ -216,6 +220,16 @@ private long getTimestampFromBlobName(String blobName) {
         return -1;
     }
 
+    private String getEntityFromBlobName(String blobName) {
+        String[] blobNameTokens = blobName.split(PINNED_TIMESTAMPS_FILENAME_SEPARATOR);
+        if (blobNameTokens.length < 2) {
+            String errorMessage = "Pinned timestamps blob name contains invalid format: " + blobName;
+            logger.error(errorMessage);
+            throw new IllegalArgumentException(errorMessage);
+        }
+        return String.join(PINNED_TIMESTAMPS_FILENAME_SEPARATOR, Arrays.copyOfRange(blobNameTokens, 0, blobNameTokens.length - 1));
+    }
+
     /**
      * Unpins a timestamp from the remote store.
      *
@@ -240,6 +254,10 @@ public void unpinTimestamp(long timestamp, String pinningEntity, ActionListener<
         }
     }
 
+    public void forceSyncPinnedTimestamps() {
+        asyncUpdatePinnedTimestampTask.run();
+    }
+
     @Override
     public void close() throws IOException {
         asyncUpdatePinnedTimestampTask.close();
@@ -258,6 +276,10 @@ public static Tuple<Long, Set<Long>> getPinnedTimestamps() {
         return pinnedTimestampsSet;
     }
 
+    public static Map<String, List<Long>> getPinnedEntities() {
+        return pinnedEntityToTimestampsMap;
+    }
+
     /**
      * Inner class for asynchronously updating the pinned timestamp set.
      */
@@ -279,6 +301,7 @@ protected void runInternal() {
                 Map<String, BlobMetadata> pinnedTimestampList = blobContainer.listBlobs();
                 if (pinnedTimestampList.isEmpty()) {
                     pinnedTimestampsSet = new Tuple<>(triggerTimestamp, Set.of());
+                    pinnedEntityToTimestampsMap = new HashMap<>();
                     return;
                 }
                 Set<Long> pinnedTimestamps = pinnedTimestampList.keySet()
@@ -286,8 +309,19 @@ protected void runInternal() {
                     .map(RemoteStorePinnedTimestampService.this::getTimestampFromBlobName)
                     .filter(timestamp -> timestamp != -1)
                     .collect(Collectors.toSet());
+
                 logger.debug("Fetched pinned timestamps from remote store: {} - {}", triggerTimestamp, pinnedTimestamps);
                 pinnedTimestampsSet = new Tuple<>(triggerTimestamp, pinnedTimestamps);
+                pinnedEntityToTimestampsMap = pinnedTimestampList.keySet()
+                    .stream()
+                    .collect(Collectors.toMap(RemoteStorePinnedTimestampService.this::getEntityFromBlobName, blobName -> {
+                        long timestamp = RemoteStorePinnedTimestampService.this.getTimestampFromBlobName(blobName);
+                        return Collections.singletonList(timestamp);
+                    }, (existingList, newList) -> {
+                        List<Long> mergedList = new ArrayList<>(existingList);
+                        mergedList.addAll(newList);
+                        return mergedList;
+                    }));
             } catch (Throwable t) {
                 logger.error("Exception while fetching pinned timestamp details", t);
             }
diff --git a/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java
new file mode 100644
index 0000000000000..ad67ecc6e9ef1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/plugins/NoOpResourceAccessControlPlugin.java
@@ -0,0 +1,88 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugins;
+
+import org.opensearch.accesscontrol.resources.EntityType;
+import org.opensearch.accesscontrol.resources.ResourceSharing;
+import org.opensearch.accesscontrol.resources.ShareWith;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This plugin class defines a no-op implementation of Resource Plugin.
+ *
+ * @opensearch.experimental
+ */
+public class NoOpResourceAccessControlPlugin implements ResourceAccessControlPlugin {
+
+    /**
+     * Returns an empty list since security plugin is not defined.
+     * This method alone doesn't determine permissions.
+     *
+     * @return empty list
+     */
+    @Override
+    public List<String> listAccessibleResourcesInPlugin(String systemIndexName) {
+        // returns an empty list since security plugin is disabled
+        return List.of();
+    }
+
+    /**
+     * @param resourceId the resource on which access is to be checked
+     * @param systemIndexName where the resource exists
+     * @param scope the type of access being requested
+     * @return true
+     */
+    @Override
+    public boolean hasPermission(String resourceId, String systemIndexName, String scope) {
+        return true;
+    }
+
+    /**
+     * @param resourceId if of the resource to be updated
+     * @param systemIndexName index where this resource is defined
+     * @param shareWith a map that contains entries of entities with whom this resource should be shared with
+     * @return null since security plugin is disabled in the cluster
+     */
+    @Override
+    public ResourceSharing shareWith(String resourceId, String systemIndexName, ShareWith shareWith) {
+        return null;
+    }
+
+    /**
+     * @param resourceId if of the resource to be updated
+     * @param systemIndexName index where this resource is defined
+     * @param revokeAccess a map that contains entries of entities whose access should be revoked
+     * @return null since security plugin is disabled in the cluster
+     */
+    @Override
+    public ResourceSharing revokeAccess(String resourceId, String systemIndexName, Map<EntityType, List<String>> revokeAccess) {
+        return null;
+    }
+
+    /**
+     * @param resourceId if of the resource to be updated
+     * @param systemIndexName index where this resource is defined
+     * @return false since security plugin is disabled
+     */
+    @Override
+    public boolean deleteResourceSharingRecord(String resourceId, String systemIndexName) {
+        return false;
+    }
+
+    /**
+     * @return false since security plugin is disabled
+     */
+    @Override
+    public boolean deleteAllResourceSharingRecordsForCurrentUser() {
+        return false;
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java
new file mode 100644
index 0000000000000..379fee4be090b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/plugins/ResourceAccessControlPlugin.java
@@ -0,0 +1,84 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugins;
+
+import org.opensearch.accesscontrol.resources.EntityType;
+import org.opensearch.accesscontrol.resources.ResourceSharing;
+import org.opensearch.accesscontrol.resources.ShareWith;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This interface determines presence of security plugin in the cluster.
+ * If yes, security plugin will be used for resource access authorization
+ * User information is fetched from thread context by security plugin.
+ * In clusters, where security plugin is disabled these requests will be pass-through via a No-op implementation.
+ * There are 3 scope of sharing for a resource: Private, Restricted, Public. To learn more visit <a href="https://github.com/opensearch-project/security/issues/4500">...</a>
+ * If security plugin is disabled, all resources will be considered public by default.
+ * TODO: add documentation around "how to use"
+ *
+ * @opensearch.experimental
+ */
+public interface ResourceAccessControlPlugin {
+
+    /**
+     * Returns all accessible resources for current user for a given system .
+     *
+     * @return list of {@link ResourceSharing} items accessible by current user.
+     */
+    List<String> listAccessibleResourcesInPlugin(String systemIndex);
+
+    /**
+     * Checks whether current user has permission to given resource.
+     *
+     * @param resourceId the resource on which access is to be checked
+     * @param systemIndexName where the resource exists
+     * @param scope the scope being requested
+     * @return true if current user has access, false otherwise
+     */
+    boolean hasPermission(String resourceId, String systemIndexName, String scope);
+
+    /**
+     * Adds an entity to the share-with. Resource needs to be in restricted mode.
+     * Creates a resource sharing record if one doesn't exist.
+     * @param resourceId id of the resource to be updated
+     * @param systemIndexName index where this resource is defined
+     * @param shareWith an object that contains entries of entities with whom the resource should be shared with
+     * @return updated resource sharing record
+     */
+    ResourceSharing shareWith(String resourceId, String systemIndexName, ShareWith shareWith);
+
+    /**
+     * Revokes given permission to a resource
+     *
+     * @param resourceId if of the resource to be updated
+     * @param systemIndexName index where this resource is defined
+     * @param revokeAccess a map that contains entries of entities whose access should be revoked
+     * @return the updated ResourceSharing record
+     */
+    ResourceSharing revokeAccess(String resourceId, String systemIndexName, Map<EntityType, List<String>> revokeAccess);
+
+    /**
+     * Deletes an entry from .resource_sharing index
+     * @param resourceId if of the resource to be updated
+     * @param systemIndexName index where this resource is defined
+     * @return true if resource record was deleted, false otherwise
+     */
+    boolean deleteResourceSharingRecord(String resourceId, String systemIndexName);
+
+    /**
+     * TODO check if this method is needed
+     * Deletes all entries from .resource_sharing index where current user is the creator of the resource
+     * @return true if resource record was deleted, false otherwise
+     */
+    boolean deleteAllResourceSharingRecordsForCurrentUser();
+
+    // TODO: Check whether methods for bulk updates are required
+}
diff --git a/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java b/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java
new file mode 100644
index 0000000000000..4fbbbf35fc367
--- /dev/null
+++ b/server/src/main/java/org/opensearch/plugins/ResourcePlugin.java
@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugins;
+
+/**
+ * This interface should be implemented by all the plugins that define one or more resources.
+ *
+ * @opensearch.experimental
+ */
+public interface ResourcePlugin {
+
+    /**
+     * Type of the resource
+     * @return a string containing the type of the resource
+     */
+    String getResourceType();
+
+    /**
+     * The index where resource meta-data is stored
+     * @return the name of the parent index where resource meta-data is stored
+     */
+    String getResourceIndex();
+}
diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java
index d4520beb5b570..138bc13140aea 100644
--- a/server/src/main/java/org/opensearch/repositories/Repository.java
+++ b/server/src/main/java/org/opensearch/repositories/Repository.java
@@ -180,7 +180,7 @@ void finalizeSnapshot(
      * @param repositoryUpdatePriority  priority for the cluster state update task
      * @param listener              listener to be invoked with the new {@link RepositoryData} after completing the snapshot
      */
-    void finalizeSnapshot(
+    default void finalizeSnapshot(
         ShardGenerations shardGenerations,
         long repositoryStateId,
         Metadata clusterMetadata,
@@ -189,7 +189,9 @@ void finalizeSnapshot(
         Function<ClusterState, ClusterState> stateTransformer,
         Priority repositoryUpdatePriority,
         ActionListener<RepositoryData> listener
-    );
+    ) {
+        throw new UnsupportedOperationException();
+    }
 
     /**
      * Deletes snapshots
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
index b954560c1bc94..14c201e819994 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
@@ -50,6 +50,7 @@
 import org.opensearch.action.ActionRunnable;
 import org.opensearch.action.StepListener;
 import org.opensearch.action.support.GroupedActionListener;
+import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateUpdateTask;
 import org.opensearch.cluster.RepositoryCleanupInProgress;
@@ -69,6 +70,7 @@
 import org.opensearch.common.Randomness;
 import org.opensearch.common.SetOnce;
 import org.opensearch.common.UUIDs;
+import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
 import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobMetadata;
 import org.opensearch.common.blobstore.BlobPath;
@@ -180,6 +182,7 @@
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
@@ -196,7 +199,6 @@
 import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1;
 import static org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName;
 import static org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS;
-import static org.opensearch.snapshots.SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER;
 
 /**
  * BlobStore - based implementation of Snapshot Repository
@@ -353,6 +355,16 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         Setting.Property.Final
     );
 
+    /**
+     * Controls the fixed prefix for the snapshot shard blob path. cluster.snapshot.async-deletion.enable
+     */
+    public static final Setting<Boolean> SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING = Setting.boolSetting(
+        "cluster.snapshot.async-deletion.enable",
+        true,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
     protected volatile boolean supportURLRepo;
 
     private volatile int maxShardBlobDeleteBatch;
@@ -446,6 +458,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private final String snapshotShardPathPrefix;
 
+    private volatile boolean enableAsyncDeletion;
+
     /**
      * Flag that is set to {@code true} if this instance is started with {@link #metadata} that has a higher value for
      * {@link RepositoryMetadata#pendingGeneration()} than for {@link RepositoryMetadata#generation()} indicating a full cluster restart
@@ -498,6 +512,8 @@ protected BlobStoreRepository(
         this.recoverySettings = recoverySettings;
         this.remoteStoreSettings = new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings());
         this.snapshotShardPathPrefix = SNAPSHOT_SHARD_PATH_PREFIX_SETTING.get(clusterService.getSettings());
+        this.enableAsyncDeletion = SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.get(clusterService.getSettings());
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING, this::setEnableAsyncDeletion);
     }
 
     @Override
@@ -1292,7 +1308,7 @@ private void cleanUpRemoteStoreFilesForDeletedIndicesV2(
             }
             // iterate through all the indices and trigger remote store directory cleanup for deleted index segments
             for (String indexId : uniqueIndexIds) {
-                cleanRemoteStoreDirectoryIfNeeded(snapshotIds, indexId, repositoryData, remoteSegmentStoreDirectoryFactory);
+                cleanRemoteStoreDirectoryIfNeeded(snapshotIds, indexId, repositoryData, remoteSegmentStoreDirectoryFactory, false);
             }
             afterCleanupsListener.onResponse(null);
         } catch (Exception e) {
@@ -1340,11 +1356,17 @@ private void removeSnapshotPinnedTimestamp(
     ) {
         remoteStorePinnedTimestampService.unpinTimestamp(
             timestampToUnpin,
-            repository + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshotId.getUUID(),
+            SnapshotsService.getPinningEntity(repository, snapshotId.getUUID()),
             new ActionListener<Void>() {
                 @Override
                 public void onResponse(Void unused) {
-                    logger.debug("Timestamp {} unpinned successfully for snapshot {}", timestampToUnpin, snapshotId.getName());
+                    logger.info("Timestamp {} unpinned successfully for snapshot {}", timestampToUnpin, snapshotId.getName());
+                    try {
+                        remoteStorePinnedTimestampService.forceSyncPinnedTimestamps();
+                        logger.debug("Successfully synced pinned timestamp state");
+                    } catch (Exception e) {
+                        logger.warn("Exception while updating pinning timestamp state, snapshot deletion will continue", e);
+                    }
                     listener.onResponse(null);
                 }
 
@@ -1449,7 +1471,8 @@ public static void remoteDirectoryCleanupAsync(
         String indexUUID,
         ShardId shardId,
         String threadPoolName,
-        RemoteStorePathStrategy pathStrategy
+        RemoteStorePathStrategy pathStrategy,
+        boolean forceClean
     ) {
         threadpool.executor(threadPoolName)
             .execute(
@@ -1459,7 +1482,8 @@ public static void remoteDirectoryCleanupAsync(
                         remoteStoreRepoForIndex,
                         indexUUID,
                         shardId,
-                        pathStrategy
+                        pathStrategy,
+                        forceClean
                     ),
                     indexUUID,
                     shardId
@@ -1515,7 +1539,8 @@ protected void releaseRemoteStoreLockAndCleanup(
                 indexUUID,
                 new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.parseInt(shardId)),
                 ThreadPool.Names.REMOTE_PURGE,
-                remoteStoreShardShallowCopySnapshot.getRemoteStorePathStrategy()
+                remoteStoreShardShallowCopySnapshot.getRemoteStorePathStrategy(),
+                false
             );
         }
     }
@@ -2078,11 +2103,11 @@ private void executeOneStaleIndexDelete(
                 deleteResult = deleteResult.add(cleanUpStaleSnapshotShardPathsFile(matchingShardPaths, snapshotShardPaths));
 
                 if (remoteSegmentStoreDirectoryFactory != null) {
-                    cleanRemoteStoreDirectoryIfNeeded(deletedSnapshots, indexSnId, oldRepoData, remoteSegmentStoreDirectoryFactory);
+                    cleanRemoteStoreDirectoryIfNeeded(deletedSnapshots, indexSnId, oldRepoData, remoteSegmentStoreDirectoryFactory, true);
                 }
 
                 // Finally, we delete the [base_path]/indexId folder
-                deleteResult = deleteResult.add(indexEntry.getValue().delete()); // Deleting the index folder
+                deleteResult = deleteResult.add(deleteContainer(indexEntry.getValue())); // Deleting the index folder
                 logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexSnId);
                 return deleteResult;
             } catch (IOException e) {
@@ -2115,6 +2140,21 @@ private void executeOneStaleIndexDelete(
         }));
     }
 
+    private DeleteResult deleteContainer(BlobContainer container) throws IOException {
+        long startTime = System.nanoTime();
+        DeleteResult deleteResult;
+        if (enableAsyncDeletion && container instanceof AsyncMultiStreamBlobContainer) {
+            // Use deleteAsync and wait for the result
+            PlainActionFuture<DeleteResult> future = new PlainActionFuture<>();
+            ((AsyncMultiStreamBlobContainer) container).deleteAsync(future);
+            deleteResult = future.actionGet();
+        } else {
+            deleteResult = container.delete();
+        }
+        logger.debug(new ParameterizedMessage("[{}] Deleted {} in {}ns", metadata.name(), container.path(), startTime - System.nanoTime()));
+        return deleteResult;
+    }
+
     /**
      * Cleans up the remote store directory if needed.
      * <p> This method cleans up segments in the remote store directory for deleted indices.
@@ -2135,7 +2175,8 @@ private void cleanRemoteStoreDirectoryIfNeeded(
         Collection<SnapshotId> deletedSnapshots,
         String indexSnId,
         RepositoryData oldRepoData,
-        RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory
+        RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory,
+        boolean forceClean
     ) {
         assert (indexSnId != null);
 
@@ -2187,9 +2228,16 @@ private void cleanRemoteStoreDirectoryIfNeeded(
                                 prevIndexMetadata.getIndexUUID(),
                                 shard,
                                 ThreadPool.Names.REMOTE_PURGE,
-                                remoteStorePathStrategy
+                                remoteStorePathStrategy,
+                                forceClean
+                            );
+                            remoteTranslogCleanupAsync(
+                                remoteTranslogRepository,
+                                shard,
+                                remoteStorePathStrategy,
+                                prevIndexMetadata,
+                                forceClean
                             );
-                            remoteTranslogCleanupAsync(remoteTranslogRepository, shard, remoteStorePathStrategy, prevIndexMetadata);
                         }
                     }
                 } catch (Exception e) {
@@ -2213,7 +2261,8 @@ private void remoteTranslogCleanupAsync(
         Repository remoteTranslogRepository,
         ShardId shardId,
         RemoteStorePathStrategy remoteStorePathStrategy,
-        IndexMetadata prevIndexMetadata
+        IndexMetadata prevIndexMetadata,
+        boolean forceClean
     ) {
         assert remoteTranslogRepository instanceof BlobStoreRepository;
         boolean indexMetadataEnabled = RemoteStoreUtils.determineTranslogMetadataEnabled(prevIndexMetadata);
@@ -2230,7 +2279,7 @@ private void remoteTranslogCleanupAsync(
             indexMetadataEnabled
         );
         try {
-            RemoteFsTimestampAwareTranslog.cleanup(translogTransferManager);
+            RemoteFsTimestampAwareTranslog.cleanupOfDeletedIndex(translogTransferManager, forceClean);
         } catch (IOException e) {
             logger.error("Exception while cleaning up remote translog for shard: " + shardId, e);
         }
@@ -2318,7 +2367,7 @@ void releaseRemoteStoreLocksAndCleanup(
      * @return A DeleteResult object representing the result of the deletion operation.
      * @throws IOException If an I/O error occurs during the deletion process.
      */
-    private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException {
+    private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException, ExecutionException, InterruptedException {
         // If the provided ShardInfo is null, return a zero DeleteResult
         if (shardInfo == null) {
             return DeleteResult.ZERO;
@@ -2330,7 +2379,7 @@ private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException {
         // Iterate over the shards and delete each shard's data
         for (int i = 0; i < shardInfo.getShardCount(); i++) {
             // Call the delete method on the shardContainer and accumulate the result
-            deleteResult = deleteResult.add(shardContainer(shardInfo.getIndexId(), i).delete());
+            deleteResult = deleteResult.add(deleteContainer(shardContainer(shardInfo.getIndexId(), i)));
         }
 
         // Return the accumulated DeleteResult
@@ -2714,7 +2763,23 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna
 
     private void deleteFromContainer(BlobContainer container, List<String> blobs) throws IOException {
         logger.trace(() -> new ParameterizedMessage("[{}] Deleting {} from [{}]", metadata.name(), blobs, container.path()));
-        container.deleteBlobsIgnoringIfNotExists(blobs);
+        long startTime = System.nanoTime();
+        if (enableAsyncDeletion && container instanceof AsyncMultiStreamBlobContainer) {
+            PlainActionFuture<Void> future = new PlainActionFuture<>();
+            ((AsyncMultiStreamBlobContainer) container).deleteBlobsAsyncIgnoringIfNotExists(blobs, future);
+            future.actionGet();
+        } else {
+            container.deleteBlobsIgnoringIfNotExists(blobs);
+        }
+        logger.debug(
+            () -> new ParameterizedMessage(
+                "[{}] Deletion {} from [{}] took {}ns",
+                metadata.name(),
+                blobs,
+                container.path(),
+                System.nanoTime() - startTime
+            )
+        );
     }
 
     private BlobPath indicesPath() {
@@ -4565,4 +4630,8 @@ public String toString() {
             return name;
         }
     }
+
+    public void setEnableAsyncDeletion(boolean enableAsyncDeletion) {
+        this.enableAsyncDeletion = enableAsyncDeletion;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java
index 0d6f965c7033f..c17f723c13f2a 100644
--- a/server/src/main/java/org/opensearch/rest/RestController.java
+++ b/server/src/main/java/org/opensearch/rest/RestController.java
@@ -41,7 +41,6 @@
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.path.PathTrie;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.common.util.io.Streams;
 import org.opensearch.common.xcontent.XContentType;
@@ -56,11 +55,6 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.http.HttpChunk;
 import org.opensearch.http.HttpServerTransport;
-import org.opensearch.identity.IdentityService;
-import org.opensearch.identity.Subject;
-import org.opensearch.identity.UserSubject;
-import org.opensearch.identity.tokens.AuthToken;
-import org.opensearch.identity.tokens.RestTokenExtractor;
 import org.opensearch.usage.UsageService;
 
 import java.io.ByteArrayOutputStream;
@@ -125,25 +119,23 @@ public class RestController implements HttpServerTransport.Dispatcher {
     /** Rest headers that are copied to internal requests made during a rest request. */
     private final Set<RestHeaderDefinition> headersToCopy;
     private final UsageService usageService;
-    private final IdentityService identityService;
 
     public RestController(
         Set<RestHeaderDefinition> headersToCopy,
         UnaryOperator<RestHandler> handlerWrapper,
         NodeClient client,
         CircuitBreakerService circuitBreakerService,
-        UsageService usageService,
-        IdentityService identityService
+        UsageService usageService
     ) {
         this.headersToCopy = headersToCopy;
         this.usageService = usageService;
         if (handlerWrapper == null) {
             handlerWrapper = h -> h; // passthrough if no wrapper set
         }
+
         this.handlerWrapper = handlerWrapper;
         this.client = client;
         this.circuitBreakerService = circuitBreakerService;
-        this.identityService = identityService;
         registerHandlerNoWrap(
             RestRequest.Method.GET,
             "/favicon.ico",
@@ -472,11 +464,6 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
                         return;
                     }
                 } else {
-                    if (FeatureFlags.isEnabled(FeatureFlags.IDENTITY)) {
-                        if (!handleAuthenticateUser(request, channel)) {
-                            return;
-                        }
-                    }
                     dispatchRequest(request, channel, handler);
                     return;
                 }
@@ -587,43 +574,6 @@ private void handleBadRequest(String uri, RestRequest.Method method, RestChannel
         }
     }
 
-    /**
-     * Attempts to extract auth token and login.
-     *
-     * @return false if there was an error and the request should not continue being dispatched
-     * */
-    private boolean handleAuthenticateUser(final RestRequest request, final RestChannel channel) {
-        try {
-            final AuthToken token = RestTokenExtractor.extractToken(request);
-            // If no token was found, continue executing the request
-            if (token == null) {
-                // Authentication did not fail so return true. Authorization is handled at the action level.
-                return true;
-            }
-            final Subject currentSubject = identityService.getCurrentSubject();
-            if (currentSubject instanceof UserSubject) {
-                ((UserSubject) currentSubject).authenticate(token);
-                logger.debug("Logged in as user " + currentSubject);
-            }
-        } catch (final Exception e) {
-            try {
-                final BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse(
-                    channel,
-                    RestStatus.UNAUTHORIZED,
-                    e.getMessage()
-                );
-                channel.sendResponse(bytesRestResponse);
-            } catch (final Exception ex) {
-                final BytesRestResponse bytesRestResponse = new BytesRestResponse(RestStatus.UNAUTHORIZED, ex.getMessage());
-                channel.sendResponse(bytesRestResponse);
-            }
-            return false;
-        }
-
-        // Authentication did not fail so return true. Authorization is handled at the action level.
-        return true;
-    }
-
     /**
      * Get the valid set of HTTP methods for a REST request.
      */
@@ -759,7 +709,7 @@ public void sendResponse(RestResponse response) {
                 prepareResponse(response.status(), Map.of("Content-Type", List.of(response.contentType())));
             }
 
-            Mono.ignoreElements(this).then(Mono.just(response)).subscribe(delegate::sendResponse);
+            Mono.from(this).ignoreElement().then(Mono.just(response)).subscribe(delegate::sendResponse);
         }
 
         @Override
diff --git a/server/src/main/java/org/opensearch/rest/RestHandler.java b/server/src/main/java/org/opensearch/rest/RestHandler.java
index 1139e5fc65f31..7c3a369be61b9 100644
--- a/server/src/main/java/org/opensearch/rest/RestHandler.java
+++ b/server/src/main/java/org/opensearch/rest/RestHandler.java
@@ -125,6 +125,13 @@ default boolean allowSystemIndexAccessByDefault() {
         return false;
     }
 
+    /**
+     * Denotes whether the RestHandler will output paginated responses or not.
+     */
+    default boolean isActionPaginated() {
+        return false;
+    }
+
     static RestHandler wrapper(RestHandler delegate) {
         return new Wrapper(delegate);
     }
@@ -185,6 +192,16 @@ public List<ReplacedRoute> replacedRoutes() {
         public boolean allowSystemIndexAccessByDefault() {
             return delegate.allowSystemIndexAccessByDefault();
         }
+
+        @Override
+        public boolean isActionPaginated() {
+            return delegate.isActionPaginated();
+        }
+
+        @Override
+        public boolean supportsStreaming() {
+            return delegate.supportsStreaming();
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java
index 2c397f7fc6e8e..f241b567c3204 100644
--- a/server/src/main/java/org/opensearch/rest/RestRequest.java
+++ b/server/src/main/java/org/opensearch/rest/RestRequest.java
@@ -51,6 +51,7 @@
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.http.HttpChannel;
 import org.opensearch.http.HttpRequest;
+import org.opensearch.rest.pagination.PageParams;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -67,6 +68,9 @@
 
 import static org.opensearch.common.unit.TimeValue.parseTimeValue;
 import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue;
+import static org.opensearch.rest.pagination.PageParams.PARAM_NEXT_TOKEN;
+import static org.opensearch.rest.pagination.PageParams.PARAM_SIZE;
+import static org.opensearch.rest.pagination.PageParams.PARAM_SORT;
 
 /**
  * REST Request
@@ -591,6 +595,10 @@ public static MediaType parseContentType(List<String> header) {
         throw new IllegalArgumentException("empty Content-Type header");
     }
 
+    public PageParams parsePaginatedQueryParams(String defaultSortOrder, int defaultPageSize) {
+        return new PageParams(param(PARAM_NEXT_TOKEN), param(PARAM_SORT, defaultSortOrder), paramAsInt(PARAM_SIZE, defaultPageSize));
+    }
+
     /**
      * Thrown if there is an error in the content type header.
      *
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java
index 6f4e060363bfb..506f5f1529776 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.common.Table;
+import org.opensearch.common.breaker.ResponseLimitSettings;
 import org.opensearch.common.io.Streams;
 import org.opensearch.common.io.UTF8StreamWriter;
 import org.opensearch.core.common.io.stream.BytesStream;
@@ -98,4 +99,12 @@ protected Set<String> responseParams() {
         return RESPONSE_PARAMS;
     }
 
+    /**
+     * Method to check if limits defined in {@link ResponseLimitSettings} are applicable to an action.
+     *
+     * @return True / False status
+     */
+    protected boolean isRequestLimitCheckSupported() {
+        return false;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java
index 9dc711f804144..7562e09ddd704 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java
@@ -50,6 +50,9 @@
 import org.opensearch.cluster.health.ClusterIndexHealth;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.Table;
+import org.opensearch.common.breaker.ResponseLimitBreachedException;
+import org.opensearch.common.breaker.ResponseLimitSettings;
+import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
@@ -61,6 +64,9 @@
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.RestResponse;
 import org.opensearch.rest.action.RestResponseListener;
+import org.opensearch.rest.action.list.AbstractListAction;
+import org.opensearch.rest.pagination.IndexPaginationStrategy;
+import org.opensearch.rest.pagination.PageToken;
 
 import java.time.Instant;
 import java.time.ZoneOffset;
@@ -68,9 +74,11 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.Spliterators;
 import java.util.function.Function;
@@ -80,6 +88,7 @@
 import static java.util.Arrays.asList;
 import static java.util.Collections.unmodifiableList;
 import static org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest.DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT;
+import static org.opensearch.common.breaker.ResponseLimitSettings.LimitEntity.INDICES;
 import static org.opensearch.rest.RestRequest.Method.GET;
 
 /**
@@ -87,7 +96,7 @@
  *
  * @opensearch.api
  */
-public class RestIndicesAction extends AbstractCatAction {
+public class RestIndicesAction extends AbstractListAction {
 
     private static final DateFormatter STRICT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_time");
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestIndicesAction.class);
@@ -96,6 +105,12 @@ public class RestIndicesAction extends AbstractCatAction {
     private static final String DUPLICATE_PARAMETER_ERROR_MESSAGE =
         "Please only use one of the request parameters [master_timeout, cluster_manager_timeout].";
 
+    private final ResponseLimitSettings responseLimitSettings;
+
+    public RestIndicesAction(ResponseLimitSettings responseLimitSettings) {
+        this.responseLimitSettings = responseLimitSettings;
+    }
+
     @Override
     public List<Route> routes() {
         return unmodifiableList(asList(new Route(GET, "/_cat/indices"), new Route(GET, "/_cat/indices/{index}")));
@@ -117,6 +132,11 @@ protected void documentation(StringBuilder sb) {
         sb.append("/_cat/indices/{index}\n");
     }
 
+    @Override
+    public boolean isRequestLimitCheckSupported() {
+        return true;
+    }
+
     @Override
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
@@ -151,48 +171,70 @@ public RestResponse buildResponse(final Table table) throws Exception {
                 new ActionListener<GetSettingsResponse>() {
                     @Override
                     public void onResponse(final GetSettingsResponse getSettingsResponse) {
-                        final GroupedActionListener<ActionResponse> groupedListener = createGroupedListener(request, 4, listener);
-                        groupedListener.onResponse(getSettingsResponse);
-
                         // The list of indices that will be returned is determined by the indices returned from the Get Settings call.
                         // All the other requests just provide additional detail, and wildcards may be resolved differently depending on the
                         // type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so
                         // force the IndicesOptions for all the sub-requests to be as inclusive as possible.
                         final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden();
-
-                        // Indices that were successfully resolved during the get settings request might be deleted when the subsequent
-                        // cluster
-                        // state, cluster health and indices stats requests execute. We have to distinguish two cases:
-                        // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the
-                        // subsequent requests to fail.
-                        // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests
-                        // not to
-                        // fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
-                        // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve
-                        // the
-                        // index names with the same indices options that we used for the initial cluster state request (strictExpand).
-                        sendIndicesStatsRequest(
-                            indices,
-                            subRequestIndicesOptions,
-                            includeUnloadedSegments,
-                            client,
-                            ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure)
-                        );
+                        // Indices that were successfully resolved during the get settings request might be deleted when the
+                        // subsequent cluster state, cluster health and indices stats requests execute. We have to distinguish two cases:
+                        // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we
+                        // want the subsequent requests to fail.
+                        // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent
+                        // requests not to fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
+                        // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests
+                        // re-resolve the index names with the same indices options that we used for the initial cluster state
+                        // request (strictExpand).
                         sendClusterStateRequest(
                             indices,
                             subRequestIndicesOptions,
                             local,
                             clusterManagerNodeTimeout,
                             client,
-                            ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure)
-                        );
-                        sendClusterHealthRequest(
-                            indices,
-                            subRequestIndicesOptions,
-                            local,
-                            clusterManagerNodeTimeout,
-                            client,
-                            ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure)
+                            new ActionListener<ClusterStateResponse>() {
+                                @Override
+                                public void onResponse(ClusterStateResponse clusterStateResponse) {
+                                    validateRequestLimit(clusterStateResponse, listener);
+                                    IndexPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse);
+                                    // For non-paginated queries, indicesToBeQueried would be same as indices retrieved from
+                                    // rest request and unresolved, while for paginated queries, it would be a list of indices
+                                    // already resolved by ClusterStateRequest and to be displayed in a page.
+                                    final String[] indicesToBeQueried = Objects.isNull(paginationStrategy)
+                                        ? indices
+                                        : paginationStrategy.getRequestedEntities().toArray(new String[0]);
+                                    final GroupedActionListener<ActionResponse> groupedListener = createGroupedListener(
+                                        request,
+                                        4,
+                                        listener,
+                                        indicesToBeQueried,
+                                        Objects.isNull(paginationStrategy) ? null : paginationStrategy.getResponseToken()
+                                    );
+                                    groupedListener.onResponse(getSettingsResponse);
+                                    groupedListener.onResponse(clusterStateResponse);
+
+                                    sendIndicesStatsRequest(
+                                        indicesToBeQueried,
+                                        subRequestIndicesOptions,
+                                        includeUnloadedSegments,
+                                        client,
+                                        ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure)
+                                    );
+
+                                    sendClusterHealthRequest(
+                                        indicesToBeQueried,
+                                        subRequestIndicesOptions,
+                                        local,
+                                        clusterManagerNodeTimeout,
+                                        client,
+                                        ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure)
+                                    );
+                                }
+
+                                @Override
+                                public void onFailure(Exception e) {
+                                    listener.onFailure(e);
+                                }
+                            }
                         );
                     }
 
@@ -203,6 +245,16 @@ public void onFailure(final Exception e) {
                 }
             );
         };
+
+    }
+
+    private void validateRequestLimit(final ClusterStateResponse clusterStateResponse, final ActionListener<Table> listener) {
+        if (isRequestLimitCheckSupported() && Objects.nonNull(clusterStateResponse) && Objects.nonNull(clusterStateResponse.getState())) {
+            int limit = responseLimitSettings.getCatIndicesResponseLimit();
+            if (ResponseLimitSettings.isResponseLimitBreached(clusterStateResponse.getState().getMetadata(), INDICES, limit)) {
+                listener.onFailure(new ResponseLimitBreachedException("Too many indices requested.", limit, INDICES));
+            }
+        }
     }
 
     /**
@@ -287,7 +339,9 @@ private void sendIndicesStatsRequest(
     private GroupedActionListener<ActionResponse> createGroupedListener(
         final RestRequest request,
         final int size,
-        final ActionListener<Table> listener
+        final ActionListener<Table> listener,
+        final String[] indicesToBeQueried,
+        final PageToken pageToken
     ) {
         return new GroupedActionListener<>(new ActionListener<Collection<ActionResponse>>() {
             @Override
@@ -311,7 +365,15 @@ public void onResponse(final Collection<ActionResponse> responses) {
                     IndicesStatsResponse statsResponse = extractResponse(responses, IndicesStatsResponse.class);
                     Map<String, IndexStats> indicesStats = statsResponse.getIndices();
 
-                    Table responseTable = buildTable(request, indicesSettings, indicesHealths, indicesStats, indicesStates);
+                    Table responseTable = buildTable(
+                        request,
+                        indicesSettings,
+                        indicesHealths,
+                        indicesStats,
+                        indicesStates,
+                        getTableIterator(indicesToBeQueried, indicesSettings),
+                        pageToken
+                    );
                     listener.onResponse(responseTable);
                 } catch (Exception e) {
                     onFailure(e);
@@ -340,7 +402,11 @@ protected Set<String> responseParams() {
 
     @Override
     protected Table getTableWithHeader(final RestRequest request) {
-        Table table = new Table();
+        return getTableWithHeader(request, null);
+    }
+
+    protected Table getTableWithHeader(final RestRequest request, final PageToken pageToken) {
+        Table table = new Table(pageToken);
         table.startHeaders();
         table.addCell("health", "alias:h;desc:current health status");
         table.addCell("status", "alias:s;desc:open/close status");
@@ -704,22 +770,27 @@ protected Table getTableWithHeader(final RestRequest request) {
     }
 
     // package private for testing
-    Table buildTable(
+    protected Table buildTable(
         final RestRequest request,
         final Map<String, Settings> indicesSettings,
         final Map<String, ClusterIndexHealth> indicesHealths,
         final Map<String, IndexStats> indicesStats,
-        final Map<String, IndexMetadata> indicesMetadatas
+        final Map<String, IndexMetadata> indicesMetadatas,
+        final Iterator<Tuple<String, Settings>> tableIterator,
+        final PageToken pageToken
     ) {
-
         final String healthParam = request.param("health");
-        final Table table = getTableWithHeader(request);
+        final Table table = getTableWithHeader(request, pageToken);
+
+        while (tableIterator.hasNext()) {
+            final Tuple<String, Settings> tuple = tableIterator.next();
+            String indexName = tuple.v1();
+            Settings settings = tuple.v2();
 
-        indicesSettings.forEach((indexName, settings) -> {
             if (indicesMetadatas.containsKey(indexName) == false) {
                 // the index exists in the Get Indices response but is not present in the cluster state:
                 // it is likely that the index was deleted in the meanwhile, so we ignore it.
-                return;
+                continue;
             }
 
             final IndexMetadata indexMetadata = indicesMetadatas.get(indexName);
@@ -748,7 +819,7 @@ Table buildTable(
                     skip = ClusterHealthStatus.RED != healthStatusFilter;
                 }
                 if (skip) {
-                    return;
+                    continue;
                 }
             }
 
@@ -982,7 +1053,8 @@ Table buildTable(
             table.addCell(searchThrottled);
 
             table.endRow();
-        });
+
+        }
 
         return table;
     }
@@ -991,4 +1063,34 @@ Table buildTable(
     private static <A extends ActionResponse> A extractResponse(final Collection<? extends ActionResponse> responses, Class<A> c) {
         return (A) responses.stream().filter(c::isInstance).findFirst().get();
     }
+
+    @Override
+    public boolean isActionPaginated() {
+        return false;
+    }
+
+    protected IndexPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) {
+        return null;
+    }
+
+    /**
+     * Provides the iterator to be used for building the response table.
+     */
+    protected Iterator<Tuple<String, Settings>> getTableIterator(String[] indices, Map<String, Settings> indexSettingsMap) {
+        return new Iterator<>() {
+            final Iterator<String> settingsMapIter = indexSettingsMap.keySet().iterator();
+
+            @Override
+            public boolean hasNext() {
+                return settingsMapIter.hasNext();
+            }
+
+            @Override
+            public Tuple<String, Settings> next() {
+                String index = settingsMapIter.next();
+                return new Tuple<>(index, indexSettingsMap.get(index));
+            }
+        };
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java
index b88af4ac3eeed..69216cb46492d 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java
@@ -42,6 +42,8 @@
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.Table;
+import org.opensearch.common.breaker.ResponseLimitBreachedException;
+import org.opensearch.common.breaker.ResponseLimitSettings;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.core.common.Strings;
 import org.opensearch.index.engine.Segment;
@@ -52,9 +54,11 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.unmodifiableList;
+import static org.opensearch.common.breaker.ResponseLimitSettings.LimitEntity.INDICES;
 import static org.opensearch.rest.RestRequest.Method.GET;
 
 /**
@@ -66,6 +70,12 @@ public class RestSegmentsAction extends AbstractCatAction {
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestSegmentsAction.class);
 
+    private final ResponseLimitSettings responseLimitSettings;
+
+    public RestSegmentsAction(ResponseLimitSettings responseLimitSettings) {
+        this.responseLimitSettings = responseLimitSettings;
+    }
+
     @Override
     public List<Route> routes() {
         return unmodifiableList(asList(new Route(GET, "/_cat/segments"), new Route(GET, "/_cat/segments/{index}")));
@@ -81,6 +91,11 @@ public boolean allowSystemIndexAccessByDefault() {
         return true;
     }
 
+    @Override
+    public boolean isRequestLimitCheckSupported() {
+        return true;
+    }
+
     @Override
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
@@ -96,6 +111,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli
         return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
+                validateRequestLimit(clusterStateResponse);
                 final IndicesSegmentsRequest indicesSegmentsRequest = new IndicesSegmentsRequest();
                 indicesSegmentsRequest.indices(indices);
                 client.admin().indices().segments(indicesSegmentsRequest, new RestResponseListener<IndicesSegmentResponse>(channel) {
@@ -110,6 +126,15 @@ public RestResponse buildResponse(final IndicesSegmentResponse indicesSegmentRes
         });
     }
 
+    private void validateRequestLimit(final ClusterStateResponse clusterStateResponse) {
+        if (isRequestLimitCheckSupported() && Objects.nonNull(clusterStateResponse) && Objects.nonNull(clusterStateResponse.getState())) {
+            int limit = responseLimitSettings.getCatSegmentsResponseLimit();
+            if (ResponseLimitSettings.isResponseLimitBreached(clusterStateResponse.getState().getRoutingTable(), INDICES, limit)) {
+                throw new ResponseLimitBreachedException("Segments from too many indices requested.", limit, INDICES);
+            }
+        }
+    }
+
     @Override
     protected void documentation(StringBuilder sb) {
         sb.append("/_cat/segments\n");
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java
index a7ad5fe6c14a3..c8b4e7472927e 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java
@@ -105,6 +105,11 @@ protected void documentation(StringBuilder sb) {
         sb.append("/_cat/shards/{index}\n");
     }
 
+    @Override
+    public boolean isRequestLimitCheckSupported() {
+        return true;
+    }
+
     @Override
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
@@ -113,6 +118,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli
         shardsRequest.clusterManagerNodeTimeout(request.paramAsTime("cluster_manager_timeout", shardsRequest.clusterManagerNodeTimeout()));
         shardsRequest.setCancelAfterTimeInterval(request.paramAsTime("cancel_after_time_interval", NO_TIMEOUT));
         shardsRequest.setIndices(indices);
+        shardsRequest.setRequestLimitCheckSupported(isRequestLimitCheckSupported());
         parseDeprecatedMasterTimeoutParameter(shardsRequest, request, deprecationLogger, getName());
         return channel -> client.execute(CatShardsAction.INSTANCE, shardsRequest, new RestResponseListener<CatShardsResponse>(channel) {
             @Override
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java
index 4f1090b163ee6..d622dd7a956f4 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java
@@ -58,8 +58,11 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
+import static org.opensearch.rest.pagination.PageToken.PAGINATED_RESPONSE_NEXT_TOKEN_KEY;
+
 /**
  * a REST table
  *
@@ -87,8 +90,37 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel
         RestRequest request = channel.request();
         XContentBuilder builder = channel.newBuilder();
         List<DisplayHeader> displayHeaders = buildDisplayHeaders(table, request);
+        if (Objects.nonNull(table.getPageToken())) {
+            buildPaginatedXContentBuilder(table, request, builder, displayHeaders);
+        } else {
+            builder.startArray();
+            addRowsToXContentBuilder(table, request, builder, displayHeaders);
+            builder.endArray();
+        }
+        return new BytesRestResponse(RestStatus.OK, builder);
+    }
+
+    private static void buildPaginatedXContentBuilder(
+        Table table,
+        RestRequest request,
+        XContentBuilder builder,
+        List<DisplayHeader> displayHeaders
+    ) throws Exception {
+        assert Objects.nonNull(table.getPageToken().getPaginatedEntity()) : "Paginated element is required in-case of paginated responses";
+        builder.startObject();
+        builder.field(PAGINATED_RESPONSE_NEXT_TOKEN_KEY, table.getPageToken().getNextToken());
+        builder.startArray(table.getPageToken().getPaginatedEntity());
+        addRowsToXContentBuilder(table, request, builder, displayHeaders);
+        builder.endArray();
+        builder.endObject();
+    }
 
-        builder.startArray();
+    private static void addRowsToXContentBuilder(
+        Table table,
+        RestRequest request,
+        XContentBuilder builder,
+        List<DisplayHeader> displayHeaders
+    ) throws Exception {
         List<Integer> rowOrder = getRowOrder(table, request);
         for (Integer row : rowOrder) {
             builder.startObject();
@@ -97,8 +129,6 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel
             }
             builder.endObject();
         }
-        builder.endArray();
-        return new BytesRestResponse(RestStatus.OK, builder);
     }
 
     public static RestResponse buildTextPlainResponse(Table table, RestChannel channel) throws IOException {
@@ -136,6 +166,11 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann
             }
             out.append("\n");
         }
+        // Adding a new row for next_token, in the response if the table is paginated.
+        if (Objects.nonNull(table.getPageToken())) {
+            out.append("next_token" + " " + table.getPageToken().getNextToken());
+            out.append("\n");
+        }
         out.close();
         return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOut.bytes());
     }
diff --git a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java
new file mode 100644
index 0000000000000..f3d6d6653a550
--- /dev/null
+++ b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java
@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rest.action.list;
+
+import org.opensearch.client.node.NodeClient;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.rest.action.cat.AbstractCatAction;
+import org.opensearch.rest.pagination.PageParams;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE;
+import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE;
+
+/**
+ * Base Transport action class for _list API.
+ * Serves as a base class for APIs wanting to support pagination.
+ * Existing _cat APIs can refer {@link org.opensearch.rest.action.cat.RestIndicesAction}.
+ * @opensearch.api
+ */
+public abstract class AbstractListAction extends AbstractCatAction {
+
+    private static final int DEFAULT_PAGE_SIZE = 100;
+    protected PageParams pageParams;
+
+    protected abstract void documentation(StringBuilder sb);
+
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        boolean helpWanted = request.paramAsBoolean("help", false);
+        if (helpWanted || isActionPaginated() == false) {
+            return super.prepareRequest(request, client);
+        }
+        this.pageParams = validateAndGetPageParams(request);
+        assert Objects.nonNull(pageParams) : "pageParams can not be null for paginated queries";
+        return doCatRequest(request, client);
+    }
+
+    @Override
+    public boolean isActionPaginated() {
+        return true;
+    }
+
+    /**
+     *
+     * @return Metadata that can be extracted out from the rest request. Query params supported by the action specific
+     * to pagination along with any respective validations to be added here.
+     */
+    protected PageParams validateAndGetPageParams(RestRequest restRequest) {
+        PageParams pageParams = restRequest.parsePaginatedQueryParams(defaultSort(), defaultPageSize());
+        // validating pageSize
+        if (pageParams.getSize() <= 0) {
+            throw new IllegalArgumentException("size must be greater than zero");
+        }
+        // Validating sort order
+        if (!(PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) || PARAM_DESC_SORT_VALUE.equals(pageParams.getSort()))) {
+            throw new IllegalArgumentException("value of sort can either be asc or desc");
+        }
+        return pageParams;
+    }
+
+    protected int defaultPageSize() {
+        return DEFAULT_PAGE_SIZE;
+    }
+
+    protected String defaultSort() {
+        return PARAM_ASC_SORT_VALUE;
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java
new file mode 100644
index 0000000000000..971cab0535be9
--- /dev/null
+++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java
@@ -0,0 +1,114 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rest.action.list;
+
+import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
+import org.opensearch.common.breaker.ResponseLimitSettings;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.rest.action.cat.RestIndicesAction;
+import org.opensearch.rest.pagination.IndexPaginationStrategy;
+import org.opensearch.rest.pagination.PageParams;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+import static org.opensearch.rest.RestRequest.Method.GET;
+
+/**
+ * _list API action to output indices in pages.
+ *
+ * @opensearch.api
+ */
+public class RestIndicesListAction extends RestIndicesAction {
+
+    private static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE = 5000;
+    private static final int DEFAULT_LIST_INDICES_PAGE_SIZE = 500;
+
+    public RestIndicesListAction(final ResponseLimitSettings responseLimitSettings) {
+        super(responseLimitSettings);
+    }
+
+    @Override
+    public List<Route> routes() {
+        return unmodifiableList(asList(new Route(GET, "/_list/indices"), new Route(GET, "/_list/indices/{index}")));
+    }
+
+    @Override
+    public String getName() {
+        return "list_indices_action";
+    }
+
+    @Override
+    protected void documentation(StringBuilder sb) {
+        sb.append("/_list/indices\n");
+        sb.append("/_list/indices/{index}\n");
+    }
+
+    @Override
+    public boolean isActionPaginated() {
+        return true;
+    }
+
+    @Override
+    protected PageParams validateAndGetPageParams(RestRequest restRequest) {
+        PageParams pageParams = super.validateAndGetPageParams(restRequest);
+        // validate max supported pageSize
+        if (pageParams.getSize() > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE) {
+            throw new IllegalArgumentException("size should be less than [" + MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE + "]");
+        }
+        // Next Token in the request will be validated by the IndexStrategyToken itself.
+        if (Objects.nonNull(pageParams.getRequestedToken())) {
+            IndexPaginationStrategy.IndexStrategyToken.validateIndexStrategyToken(pageParams.getRequestedToken());
+        }
+        return pageParams;
+    }
+
+    @Override
+    public boolean isRequestLimitCheckSupported() {
+        return false;
+    }
+
+    protected int defaultPageSize() {
+        return DEFAULT_LIST_INDICES_PAGE_SIZE;
+    }
+
+    @Override
+    protected IndexPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) {
+        return new IndexPaginationStrategy(pageParams, clusterStateResponse.getState());
+    }
+
+    // Public for testing
+    @Override
+    public Iterator<Tuple<String, Settings>> getTableIterator(String[] indices, Map<String, Settings> indexSettingsMap) {
+        return new Iterator<>() {
+            int indexPos = 0;
+
+            @Override
+            public boolean hasNext() {
+                while (indexPos < indices.length && indexSettingsMap.containsKey(indices[indexPos]) == false) {
+                    indexPos++;
+                }
+                return indexPos < indices.length;
+            }
+
+            @Override
+            public Tuple<String, Settings> next() {
+                Tuple<String, Settings> tuple = new Tuple<>(indices[indexPos], indexSettingsMap.get(indices[indexPos]));
+                indexPos++;
+                return tuple;
+            }
+        };
+    }
+}
diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java
new file mode 100644
index 0000000000000..4b8551ea7e14a
--- /dev/null
+++ b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rest.action.list;
+
+import org.opensearch.client.node.NodeClient;
+import org.opensearch.core.rest.RestStatus;
+import org.opensearch.rest.BaseRestHandler;
+import org.opensearch.rest.BytesRestResponse;
+import org.opensearch.rest.RestRequest;
+
+import java.io.IOException;
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.opensearch.rest.RestRequest.Method.GET;
+
+/**
+ * Base _list API endpoint
+ *
+ * @opensearch.api
+ */
+public class RestListAction extends BaseRestHandler {
+
+    private static final String LIST = ":‑|";
+    private static final String LIST_NL = LIST + "\n";
+    private final String HELP;
+
+    public RestListAction(List<AbstractListAction> listActions) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(LIST_NL);
+        for (AbstractListAction listAction : listActions) {
+            listAction.documentation(sb);
+        }
+        HELP = sb.toString();
+    }
+
+    @Override
+    public List<Route> routes() {
+        return singletonList(new Route(GET, "/_list"));
+    }
+
+    @Override
+    public String getName() {
+        return "list_action";
+    }
+
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, HELP));
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/rest/action/list/package-info.java b/server/src/main/java/org/opensearch/rest/action/list/package-info.java
new file mode 100644
index 0000000000000..8d6563ff9b344
--- /dev/null
+++ b/server/src/main/java/org/opensearch/rest/action/list/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * {@link org.opensearch.rest.RestHandler}s for actions that list out results in chunks of pages.
+ */
+package org.opensearch.rest.action.list;
diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java
index 3a6b45013e892..05465e32631fd 100644
--- a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java
@@ -210,7 +210,7 @@ public static void parseSearchRequest(
         searchRequest.routing(request.param("routing"));
         searchRequest.preference(request.param("preference"));
         searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
-        searchRequest.pipeline(request.param("search_pipeline"));
+        searchRequest.pipeline(request.param("search_pipeline", searchRequest.source().pipeline()));
 
         checkRestTotalHits(request, searchRequest);
         request.paramAsBoolean(INCLUDE_NAMED_QUERIES_SCORE_PARAM, false);
diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java
new file mode 100644
index 0000000000000..f89ab14e4b24d
--- /dev/null
+++ b/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java
@@ -0,0 +1,185 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rest.pagination;
+
+import org.opensearch.OpenSearchParseException;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.Nullable;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE;
+
+/**
+ * This strategy can be used by the Rest APIs wanting to paginate the responses based on Indices.
+ * The strategy considers create timestamps of indices as the keys to iterate over pages.
+ *
+ * @opensearch.internal
+ */
+public class IndexPaginationStrategy implements PaginationStrategy<String> {
+    private static final String DEFAULT_INDICES_PAGINATED_ENTITY = "indices";
+
+    private static final Comparator<IndexMetadata> ASC_COMPARATOR = (metadata1, metadata2) -> {
+        if (metadata1.getCreationDate() == metadata2.getCreationDate()) {
+            return metadata1.getIndex().getName().compareTo(metadata2.getIndex().getName());
+        }
+        return Long.compare(metadata1.getCreationDate(), metadata2.getCreationDate());
+    };
+    private static final Comparator<IndexMetadata> DESC_COMPARATOR = (metadata1, metadata2) -> {
+        if (metadata1.getCreationDate() == metadata2.getCreationDate()) {
+            return metadata2.getIndex().getName().compareTo(metadata1.getIndex().getName());
+        }
+        return Long.compare(metadata2.getCreationDate(), metadata1.getCreationDate());
+    };
+
+    private final PageToken pageToken;
+    private final List<String> requestedIndices;
+
+    public IndexPaginationStrategy(PageParams pageParams, ClusterState clusterState) {
+        // Get list of indices metadata sorted by their creation time and filtered by the last sent index
+        List<IndexMetadata> sortedIndices = PaginationStrategy.getSortedIndexMetadata(
+            clusterState,
+            getMetadataFilter(pageParams.getRequestedToken(), pageParams.getSort()),
+            PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) ? ASC_COMPARATOR : DESC_COMPARATOR
+        );
+        // Trim sortedIndicesList to get the list of indices metadata to be sent as response
+        List<IndexMetadata> metadataSublist = getMetadataSubList(sortedIndices, pageParams.getSize());
+        // Get list of index names from the trimmed metadataSublist
+        this.requestedIndices = metadataSublist.stream().map(metadata -> metadata.getIndex().getName()).collect(Collectors.toList());
+        this.pageToken = getResponseToken(
+            pageParams.getSize(),
+            sortedIndices.size(),
+            metadataSublist.isEmpty() ? null : metadataSublist.get(metadataSublist.size() - 1)
+        );
+    }
+
+    private static Predicate<IndexMetadata> getMetadataFilter(String requestedTokenStr, String sortOrder) {
+        boolean isAscendingSort = sortOrder.equals(PARAM_ASC_SORT_VALUE);
+        IndexStrategyToken requestedToken = Objects.isNull(requestedTokenStr) || requestedTokenStr.isEmpty()
+            ? null
+            : new IndexStrategyToken(requestedTokenStr);
+        if (Objects.isNull(requestedToken)) {
+            return indexMetadata -> true;
+        }
+        return metadata -> {
+            if (metadata.getIndex().getName().equals(requestedToken.lastIndexName)) {
+                return false;
+            } else if (metadata.getCreationDate() == requestedToken.lastIndexCreationTime) {
+                return isAscendingSort
+                    ? metadata.getIndex().getName().compareTo(requestedToken.lastIndexName) > 0
+                    : metadata.getIndex().getName().compareTo(requestedToken.lastIndexName) < 0;
+            }
+            return isAscendingSort
+                ? metadata.getCreationDate() > requestedToken.lastIndexCreationTime
+                : metadata.getCreationDate() < requestedToken.lastIndexCreationTime;
+        };
+    }
+
+    private List<IndexMetadata> getMetadataSubList(List<IndexMetadata> sortedIndices, final int pageSize) {
+        if (sortedIndices.isEmpty()) {
+            return new ArrayList<>();
+        }
+        return sortedIndices.subList(0, Math.min(pageSize, sortedIndices.size()));
+    }
+
+    private PageToken getResponseToken(final int pageSize, final int totalIndices, IndexMetadata lastIndex) {
+        if (totalIndices <= pageSize) {
+            return new PageToken(null, DEFAULT_INDICES_PAGINATED_ENTITY);
+        }
+        return new PageToken(
+            new IndexStrategyToken(lastIndex.getCreationDate(), lastIndex.getIndex().getName()).generateEncryptedToken(),
+            DEFAULT_INDICES_PAGINATED_ENTITY
+        );
+    }
+
+    @Override
+    @Nullable
+    public PageToken getResponseToken() {
+        return pageToken;
+    }
+
+    @Override
+    public List<String> getRequestedEntities() {
+        return Objects.isNull(requestedIndices) ? new ArrayList<>() : requestedIndices;
+    }
+
+    /**
+     * TokenParser to be used by {@link IndexPaginationStrategy}.
+     * Token would look like: CreationTimeOfLastRespondedIndex + | + NameOfLastRespondedIndex
+     */
+    public static class IndexStrategyToken {
+
+        private static final String JOIN_DELIMITER = "|";
+        private static final String SPLIT_REGEX = "\\|";
+        private static final int CREATE_TIME_POS_IN_TOKEN = 0;
+        private static final int INDEX_NAME_POS_IN_TOKEN = 1;
+
+        /**
+         * Represents creation times of last index which was displayed in the page.
+         * Used to identify the new start point in case the indices get created/deleted while queries are executed.
+         */
+        private final long lastIndexCreationTime;
+
+        /**
+         * Represents name of the last index which was displayed in the page.
+         * Used to identify whether the sorted list of indices has changed or not.
+         */
+        private final String lastIndexName;
+
+        public IndexStrategyToken(String requestedTokenString) {
+            // TODO: Avoid validating the requested token multiple times while calling from Rest and/or Transport layer.
+            validateIndexStrategyToken(requestedTokenString);
+            String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString);
+            final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX);
+            this.lastIndexCreationTime = Long.parseLong(decryptedTokenElements[CREATE_TIME_POS_IN_TOKEN]);
+            this.lastIndexName = decryptedTokenElements[INDEX_NAME_POS_IN_TOKEN];
+        }
+
+        public IndexStrategyToken(long creationTimeOfLastRespondedIndex, String nameOfLastRespondedIndex) {
+            Objects.requireNonNull(nameOfLastRespondedIndex, "index name should be provided");
+            this.lastIndexCreationTime = creationTimeOfLastRespondedIndex;
+            this.lastIndexName = nameOfLastRespondedIndex;
+        }
+
+        public String generateEncryptedToken() {
+            return PaginationStrategy.encryptStringToken(String.join(JOIN_DELIMITER, String.valueOf(lastIndexCreationTime), lastIndexName));
+        }
+
+        /**
+         * Will perform simple validations on token received in the request.
+         * Token should be base64 encoded, and should contain the expected number of elements separated by "|".
+         * Timestamps should also be a valid long.
+         *
+         * @param requestedTokenStr string denoting the encoded token requested by the user.
+         */
+        public static void validateIndexStrategyToken(String requestedTokenStr) {
+            Objects.requireNonNull(requestedTokenStr, "requestedTokenString can not be null");
+            String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenStr);
+            final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX);
+            if (decryptedTokenElements.length != 2) {
+                throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE);
+            }
+            try {
+                long creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[CREATE_TIME_POS_IN_TOKEN]);
+                if (creationTimeOfLastRespondedIndex <= 0) {
+                    throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE);
+                }
+            } catch (NumberFormatException exception) {
+                throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE);
+            }
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageParams.java b/server/src/main/java/org/opensearch/rest/pagination/PageParams.java
new file mode 100644
index 0000000000000..9b2074bc3fed0
--- /dev/null
+++ b/server/src/main/java/org/opensearch/rest/pagination/PageParams.java
@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rest.pagination;
+
+import org.opensearch.common.annotation.PublicApi;
+
+/**
+ *
+ * Class specific to paginated queries, which will contain common query params required by a paginated API.
+ */
+@PublicApi(since = "3.0.0")
+public class PageParams {
+
+    public static final String PARAM_SORT = "sort";
+    public static final String PARAM_NEXT_TOKEN = "next_token";
+    public static final String PARAM_SIZE = "size";
+    public static final String PARAM_ASC_SORT_VALUE = "asc";
+    public static final String PARAM_DESC_SORT_VALUE = "desc";
+
+    private final String requestedTokenStr;
+    private final String sort;
+    private final int size;
+
+    public PageParams(String requestedToken, String sort, int size) {
+        this.requestedTokenStr = requestedToken;
+        this.sort = sort;
+        this.size = size;
+    }
+
+    public String getSort() {
+        return sort;
+    }
+
+    public String getRequestedToken() {
+        return requestedTokenStr;
+    }
+
+    public int getSize() {
+        return size;
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageToken.java b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java
new file mode 100644
index 0000000000000..d62e1be695715
--- /dev/null
+++ b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rest.pagination;
+
+/**
+ * Pagination response metadata for a paginated query.
+ * @opensearch.internal
+ */
+public class PageToken {
+
+    public static final String PAGINATED_RESPONSE_NEXT_TOKEN_KEY = "next_token";
+
+    /**
+     * String denoting the next_token of paginated response, which will be used to fetch next page (if any).
+     */
+    private final String nextToken;
+
+    /**
+     * String denoting the element which is being paginated (for e.g. shards, indices..).
+     */
+    private final String paginatedEntity;
+
+    public PageToken(String nextToken, String paginatedElement) {
+        assert paginatedElement != null : "paginatedElement must be specified for a paginated response";
+        this.nextToken = nextToken;
+        this.paginatedEntity = paginatedElement;
+    }
+
+    public String getNextToken() {
+        return nextToken;
+    }
+
+    public String getPaginatedEntity() {
+        return paginatedEntity;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java
new file mode 100644
index 0000000000000..7f9825a7cc09b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java
@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rest.pagination;
+
+import org.opensearch.OpenSearchParseException;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.IndexMetadata;
+
+import java.util.Base64;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Interface to be implemented by any strategy getting used for paginating rest responses.
+ *
+ * @opensearch.internal
+ */
+public interface PaginationStrategy<T> {
+
+    String INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE =
+        "Parameter [next_token] has been tainted and is incorrect. Please provide a valid [next_token].";
+
+    /**
+     *
+     * @return Base64 encoded string, which can be used to fetch next page of response.
+     */
+    PageToken getResponseToken();
+
+    /**
+     *
+     * @return List of elements fetched corresponding to the store and token received by the strategy.
+     */
+    List<T> getRequestedEntities();
+
+    /**
+     *
+     * Utility method to get list of indices filtered as per {@param filterPredicate} and the sorted according to {@param comparator}.
+     */
+    static List<IndexMetadata> getSortedIndexMetadata(
+        final ClusterState clusterState,
+        Predicate<IndexMetadata> filterPredicate,
+        Comparator<IndexMetadata> comparator
+    ) {
+        return clusterState.metadata().indices().values().stream().filter(filterPredicate).sorted(comparator).collect(Collectors.toList());
+    }
+
+    static String encryptStringToken(String tokenString) {
+        if (Objects.isNull(tokenString)) {
+            return null;
+        }
+        return Base64.getEncoder().encodeToString(tokenString.getBytes(UTF_8));
+    }
+
+    static String decryptStringToken(String encTokenString) {
+        if (Objects.isNull(encTokenString)) {
+            return null;
+        }
+        try {
+            return new String(Base64.getDecoder().decode(encTokenString), UTF_8);
+        } catch (IllegalArgumentException exception) {
+            throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/rest/pagination/package-info.java b/server/src/main/java/org/opensearch/rest/pagination/package-info.java
new file mode 100644
index 0000000000000..324b8a6c46f88
--- /dev/null
+++ b/server/src/main/java/org/opensearch/rest/pagination/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Exposes utilities for Rest actions to paginate responses.
+ */
+package org.opensearch.rest.pagination;
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java
index 150efa878f866..db8979d611b4f 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java
@@ -46,7 +46,6 @@
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.lucene.search.Queries;
 import org.opensearch.core.ParseField;
-import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.ObjectMapper;
 import org.opensearch.search.aggregations.Aggregator;
 import org.opensearch.search.aggregations.AggregatorFactories;
@@ -63,6 +62,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.opensearch.index.mapper.ObjectMapper.Nested.isParent;
+
 /**
  * Aggregate all docs that match a nested path
  *
@@ -98,17 +99,6 @@ public class NestedAggregator extends BucketsAggregator implements SingleBucketA
         this.collectsFromSingleBucket = cardinality.map(estimate -> estimate < 2);
     }
 
-    private boolean isParent(ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, MapperService mapperService) {
-        if (parentObjectMapper == null) {
-            return false;
-        }
-        ObjectMapper parent;
-        do {
-            parent = childObjectMapper.getParentObjectMapper(mapperService);
-        } while (parent != null && parent != parentObjectMapper);
-        return parentObjectMapper == parent;
-    }
-
     @Override
     public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException {
         IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(ctx);
diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index 7a73fafb4a809..1ccceb1d77dcb 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -40,6 +40,7 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.query.WithFieldName;
 import org.opensearch.script.Script;
 import org.opensearch.search.aggregations.AbstractAggregationBuilder;
 import org.opensearch.search.aggregations.AggregationInitializationException;
@@ -57,7 +58,9 @@
  *
  * @opensearch.internal
  */
-public abstract class ValuesSourceAggregationBuilder<AB extends ValuesSourceAggregationBuilder<AB>> extends AbstractAggregationBuilder<AB> {
+public abstract class ValuesSourceAggregationBuilder<AB extends ValuesSourceAggregationBuilder<AB>> extends AbstractAggregationBuilder<AB>
+    implements
+        WithFieldName {
 
     public static <T> void declareFields(
         AbstractObjectParser<? extends ValuesSourceAggregationBuilder<?>, T> objectParser,
@@ -292,6 +295,11 @@ public String field() {
         return field;
     }
 
+    @Override
+    public String fieldName() {
+        return field();
+    }
+
     /**
      * Sets the script to use for this aggregation.
      */
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
index 43b9f8ae87529..e98046ba1dede 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
@@ -158,14 +158,16 @@ public SearchBackpressureService(
                 timeNanosSupplier,
                 getSettings().getSearchTaskSettings().getCancellationRateNanos(),
                 getSettings().getSearchTaskSettings().getCancellationBurst(),
-                getSettings().getSearchTaskSettings().getCancellationRatio()
+                getSettings().getSearchTaskSettings().getCancellationRatio(),
+                getSettings().getSearchTaskSettings().getCancellationRate()
             ),
             SearchShardTask.class,
             new SearchBackpressureState(
                 timeNanosSupplier,
                 getSettings().getSearchShardTaskSettings().getCancellationRateNanos(),
                 getSettings().getSearchShardTaskSettings().getCancellationBurst(),
-                getSettings().getSearchShardTaskSettings().getCancellationRatio()
+                getSettings().getSearchShardTaskSettings().getCancellationRatio(),
+                getSettings().getSearchShardTaskSettings().getCancellationRate()
             )
         );
         this.settings.getSearchTaskSettings().addListener(searchBackpressureStates.get(SearchTask.class));
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java
index 5f086bd498036..36f5b25e002c3 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java
@@ -43,12 +43,15 @@ public class SearchBackpressureState implements CancellationSettingsListener {
         LongSupplier timeNanosSupplier,
         double cancellationRateNanos,
         double cancellationBurst,
-        double cancellationRatio
+        double cancellationRatio,
+        double cancellationRate
     ) {
         rateLimiter = new AtomicReference<>(new TokenBucket(timeNanosSupplier, cancellationRateNanos, cancellationBurst));
         ratioLimiter = new AtomicReference<>(new TokenBucket(this::getCompletionCount, cancellationRatio, cancellationBurst));
         this.timeNanosSupplier = timeNanosSupplier;
         this.cancellationBurst = cancellationBurst;
+        this.cancellationRatio = cancellationRatio;
+        this.cancellationRate = cancellationRate;
     }
 
     public long getCompletionCount() {
diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java
index 79494eb0d3c24..55a031382f282 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java
@@ -61,8 +61,14 @@ private static class Defaults {
     public static final Setting<Double> SETTING_CANCELLATION_RATIO = Setting.doubleSetting(
         "search_backpressure.cancellation_ratio",
         Defaults.CANCELLATION_RATIO,
-        0.0,
-        1.0,
+        value -> {
+            if (value <= 0.0) {
+                throw new IllegalArgumentException("search_backpressure.cancellation_ratio must be > 0");
+            }
+            if (value > 1.0) {
+                throw new IllegalArgumentException("search_backpressure.cancellation_ratio must be <= 1.0");
+            }
+        },
         Setting.Property.Deprecated,
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
@@ -78,7 +84,11 @@ private static class Defaults {
     public static final Setting<Double> SETTING_CANCELLATION_RATE = Setting.doubleSetting(
         "search_backpressure.cancellation_rate",
         Defaults.CANCELLATION_RATE,
-        0.0,
+        value -> {
+            if (value <= 0.0) {
+                throw new IllegalArgumentException("search_backpressure.cancellation_rate must be > 0");
+            }
+        },
         Setting.Property.Deprecated,
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java
index 6d016c7466362..38213506c55b7 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java
@@ -44,8 +44,14 @@ private static class Defaults {
     public static final Setting<Double> SETTING_CANCELLATION_RATIO = Setting.doubleSetting(
         "search_backpressure.search_shard_task.cancellation_ratio",
         SearchBackpressureSettings.SETTING_CANCELLATION_RATIO,
-        0.0,
-        1.0,
+        value -> {
+            if (value <= 0.0) {
+                throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_ratio must be > 0");
+            }
+            if (value > 1.0) {
+                throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_ratio must be <= 1.0");
+            }
+        },
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
     );
@@ -58,7 +64,11 @@ private static class Defaults {
     public static final Setting<Double> SETTING_CANCELLATION_RATE = Setting.doubleSetting(
         "search_backpressure.search_shard_task.cancellation_rate",
         SearchBackpressureSettings.SETTING_CANCELLATION_RATE,
-        0.0,
+        value -> {
+            if (value <= 0.0) {
+                throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_rate must be > 0");
+            }
+        },
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
     );
diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java
index 4b34323b1ddc6..f9af7f9b59fdb 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java
@@ -48,8 +48,14 @@ private static class Defaults {
     public static final Setting<Double> SETTING_CANCELLATION_RATIO = Setting.doubleSetting(
         "search_backpressure.search_task.cancellation_ratio",
         Defaults.CANCELLATION_RATIO,
-        0.0,
-        1.0,
+        value -> {
+            if (value <= 0.0) {
+                throw new IllegalArgumentException("search_backpressure.search_task.cancellation_ratio must be > 0");
+            }
+            if (value > 1.0) {
+                throw new IllegalArgumentException("search_backpressure.search_task.cancellation_ratio must be <= 1.0");
+            }
+        },
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
     );
@@ -62,7 +68,11 @@ private static class Defaults {
     public static final Setting<Double> SETTING_CANCELLATION_RATE = Setting.doubleSetting(
         "search_backpressure.search_task.cancellation_rate",
         Defaults.CANCELLATION_RATE,
-        0.0,
+        value -> {
+            if (value <= 0.0) {
+                throw new IllegalArgumentException("search_backpressure.search_task.cancellation_rate must be > 0");
+            }
+        },
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
     );
diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
index 8a9704b04566f..9c438401b9fbe 100644
--- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
@@ -224,6 +224,8 @@ public static HighlightBuilder highlight() {
 
     private Map<String, Object> searchPipelineSource = null;
 
+    private String searchPipeline;
+
     /**
      * Constructs a new search source builder.
      */
@@ -297,6 +299,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException {
                 derivedFields = in.readList(DerivedField::new);
             }
         }
+        if (in.getVersion().onOrAfter(Version.V_2_18_0)) {
+            searchPipeline = in.readOptionalString();
+        }
     }
 
     @Override
@@ -377,6 +382,9 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeList(derivedFields);
             }
         }
+        if (out.getVersion().onOrAfter(Version.V_2_18_0)) {
+            out.writeOptionalString(searchPipeline);
+        }
     }
 
     /**
@@ -1111,6 +1119,13 @@ public Map<String, Object> searchPipelineSource() {
         return searchPipelineSource;
     }
 
+    /**
+     * @return a search pipeline name defined within the search source (see {@link org.opensearch.search.pipeline.SearchPipelineService})
+     */
+    public String pipeline() {
+        return searchPipeline;
+    }
+
     /**
      * Define a search pipeline to process this search request and/or its response. See {@link org.opensearch.search.pipeline.SearchPipelineService}.
      */
@@ -1119,6 +1134,14 @@ public SearchSourceBuilder searchPipelineSource(Map<String, Object> searchPipeli
         return this;
     }
 
+    /**
+     * Define a search pipeline name to process this search request and/or its response. See {@link org.opensearch.search.pipeline.SearchPipelineService}.
+     */
+    public SearchSourceBuilder pipeline(String searchPipeline) {
+        this.searchPipeline = searchPipeline;
+        return this;
+    }
+
     /**
      * Rewrites this search source builder into its primitive form. e.g. by
      * rewriting the QueryBuilder. If the builder did not change the identity
@@ -1216,6 +1239,7 @@ private SearchSourceBuilder shallowCopy(
         rewrittenBuilder.pointInTimeBuilder = pointInTimeBuilder;
         rewrittenBuilder.derivedFieldsObject = derivedFieldsObject;
         rewrittenBuilder.derivedFields = derivedFields;
+        rewrittenBuilder.searchPipeline = searchPipeline;
         return rewrittenBuilder;
     }
 
@@ -1283,6 +1307,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th
                     sort(parser.text());
                 } else if (PROFILE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     profile = parser.booleanValue();
+                } else if (SEARCH_PIPELINE.match(currentFieldName, parser.getDeprecationHandler())) {
+                    searchPipeline = parser.text();
                 } else {
                     throw new ParsingException(
                         parser.getTokenLocation(),
@@ -1612,6 +1638,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
 
         }
 
+        if (searchPipeline != null) {
+            builder.field(SEARCH_PIPELINE.getPreferredName(), searchPipeline);
+        }
+
         return builder;
     }
 
@@ -1889,7 +1919,8 @@ public int hashCode() {
             trackTotalHitsUpTo,
             pointInTimeBuilder,
             derivedFieldsObject,
-            derivedFields
+            derivedFields,
+            searchPipeline
         );
     }
 
@@ -1934,7 +1965,8 @@ public boolean equals(Object obj) {
             && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo)
             && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder)
             && Objects.equals(derivedFieldsObject, other.derivedFieldsObject)
-            && Objects.equals(derivedFields, other.derivedFields);
+            && Objects.equals(derivedFields, other.derivedFields)
+            && Objects.equals(searchPipeline, other.searchPipeline);
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java
index 5cecda1346b90..9825b2cbbe08e 100644
--- a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java
+++ b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java
@@ -65,6 +65,7 @@
 import org.opensearch.index.query.QueryRewriteContext;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.query.QueryShardException;
+import org.opensearch.index.query.WithFieldName;
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.MultiValueMode;
 import org.opensearch.search.SearchSortValuesAndFormats;
@@ -86,7 +87,7 @@
  *
  * @opensearch.internal
  */
-public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
+public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> implements WithFieldName {
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(FieldSortBuilder.class);
 
     public static final String NAME = "field_sort";
@@ -184,6 +185,11 @@ public String getFieldName() {
         return this.fieldName;
     }
 
+    @Override
+    public String fieldName() {
+        return getFieldName();
+    }
+
     /**
      * Sets the value when a field is missing in a doc. Can also be set to {@code _last} or
      * {@code _first} to sort missing last or first respectively.
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
index f6e550525a3e5..541a87c200883 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
@@ -38,6 +38,7 @@
 import org.opensearch.ExceptionsHelper;
 import org.opensearch.Version;
 import org.opensearch.action.ActionRunnable;
+import org.opensearch.action.LatchedActionListener;
 import org.opensearch.action.StepListener;
 import org.opensearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest;
 import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
@@ -381,7 +382,7 @@ public ClusterState execute(ClusterState currentState) {
                 );
                 if (request.partial() == false) {
                     Set<String> missing = new HashSet<>();
-                    for (final Map.Entry<ShardId, SnapshotsInProgress.ShardSnapshotStatus> entry : shards.entrySet()) {
+                    for (final Map.Entry<ShardId, ShardSnapshotStatus> entry : shards.entrySet()) {
                         if (entry.getValue().state() == ShardState.MISSING) {
                             missing.add(entry.getKey().getIndex().getName());
                         }
@@ -468,26 +469,33 @@ public void createSnapshotV2(final CreateSnapshotRequest request, final ActionLi
         long pinnedTimestamp = System.currentTimeMillis();
         final String repositoryName = request.repository();
         final String snapshotName = indexNameExpressionResolver.resolveDateMathExpression(request.snapshot());
-        validate(repositoryName, snapshotName);
 
-        final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot
         Repository repository = repositoriesService.repository(repositoryName);
+        validate(repositoryName, snapshotName);
+        repository.executeConsistentStateUpdate(repositoryData -> new ClusterStateUpdateTask(Priority.URGENT) {
+            private SnapshotsInProgress.Entry newEntry;
 
-        if (repository.isReadOnly()) {
-            listener.onFailure(
-                new RepositoryException(repository.getMetadata().name(), "cannot create snapshot-v2 in a readonly repository")
-            );
-            return;
-        }
+            private SnapshotId snapshotId;
 
-        final Snapshot snapshot = new Snapshot(repositoryName, snapshotId);
-        ClusterState currentState = clusterService.state();
-        final Map<String, Object> userMeta = repository.adaptUserMetadata(request.userMetadata());
-        try {
-            final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
-            repositoriesService.getRepositoryData(repositoryName, repositoryDataListener);
+            private Snapshot snapshot;
+
+            boolean enteredLoop;
+
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                // move to in progress
+                snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot
+                Repository repository = repositoriesService.repository(repositoryName);
+
+                if (repository.isReadOnly()) {
+                    listener.onFailure(
+                        new RepositoryException(repository.getMetadata().name(), "cannot create snapshot-v2 in a readonly repository")
+                    );
+                }
+
+                snapshot = new Snapshot(repositoryName, snapshotId);
+                final Map<String, Object> userMeta = repository.adaptUserMetadata(request.userMetadata());
 
-            repositoryDataListener.whenComplete(repositoryData -> {
                 createSnapshotPreValidations(currentState, repositoryData, repositoryName, snapshotName);
 
                 List<String> indices = new ArrayList<>(currentState.metadata().indices().keySet());
@@ -498,7 +506,7 @@ public void createSnapshotV2(final CreateSnapshotRequest request, final ActionLi
                     request.indices()
                 );
 
-                logger.trace("[{}][{}] creating snapshot-v2 for indices [{}]", repositoryName, snapshotName, indices);
+                logger.info("[{}][{}] creating snapshot-v2 for indices [{}]", repositoryName, snapshotName, indices);
 
                 final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
                 final List<SnapshotsInProgress.Entry> runningSnapshots = snapshots.entries();
@@ -509,73 +517,193 @@ public void createSnapshotV2(final CreateSnapshotRequest request, final ActionLi
                     IndexId.DEFAULT_SHARD_PATH_TYPE
                 );
                 final Version version = minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null);
-                final ShardGenerations shardGenerations = buildShardsGenerationFromRepositoryData(
-                    currentState.metadata(),
-                    currentState.routingTable(),
-                    indexIds,
-                    repositoryData
-                );
 
                 if (repositoryData.getGenId() == RepositoryData.UNKNOWN_REPO_GEN) {
                     logger.debug("[{}] was aborted before starting", snapshot);
                     throw new SnapshotException(snapshot, "Aborted on initialization");
                 }
+
+                Map<ShardId, ShardSnapshotStatus> shards = new HashMap<>();
+
+                newEntry = SnapshotsInProgress.startedEntry(
+                    new Snapshot(repositoryName, snapshotId),
+                    request.includeGlobalState(),
+                    request.partial(),
+                    indexIds,
+                    dataStreams,
+                    threadPool.absoluteTimeInMillis(),
+                    repositoryData.getGenId(),
+                    shards,
+                    userMeta,
+                    version,
+                    true,
+                    true
+                );
+                final List<SnapshotsInProgress.Entry> newEntries = new ArrayList<>(runningSnapshots);
+                newEntries.add(newEntry);
+
+                // Entering finalize loop here to prevent concurrent snapshots v2 snapshots
+                enteredLoop = tryEnterRepoLoop(repositoryName);
+                if (enteredLoop == false) {
+                    throw new ConcurrentSnapshotExecutionException(
+                        repositoryName,
+                        snapshotName,
+                        "cannot start snapshot-v2 while a repository is in finalization state"
+                    );
+                }
+                return ClusterState.builder(currentState)
+                    .putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(new ArrayList<>(newEntries)))
+                    .build();
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to create snapshot-v2", repositoryName, snapshotName), e);
+                listener.onFailure(e);
+                if (enteredLoop) {
+                    leaveRepoLoop(repositoryName);
+                }
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) {
+                final ShardGenerations shardGenerations = buildShardsGenerationFromRepositoryData(
+                    newState.metadata(),
+                    newState.routingTable(),
+                    newEntry.indices(),
+                    repositoryData
+                );
+                final List<String> dataStreams = indexNameExpressionResolver.dataStreamNames(
+                    newState,
+                    request.indicesOptions(),
+                    request.indices()
+                );
                 final SnapshotInfo snapshotInfo = new SnapshotInfo(
-                    snapshot.getSnapshotId(),
+                    snapshotId,
                     shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()),
-                    dataStreams,
+                    newEntry.dataStreams(),
                     pinnedTimestamp,
                     null,
                     System.currentTimeMillis(),
                     shardGenerations.totalShards(),
                     Collections.emptyList(),
                     request.includeGlobalState(),
-                    userMeta,
+                    newEntry.userMetadata(),
                     true,
                     pinnedTimestamp
                 );
-                if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) {
-                    throw new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2, no longer cluster manager");
-                }
+                final Version version = minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null);
                 final StepListener<RepositoryData> pinnedTimestampListener = new StepListener<>();
-                pinnedTimestampListener.whenComplete(repoData -> { listener.onResponse(snapshotInfo); }, listener::onFailure);
-                repository.finalizeSnapshot(
-                    shardGenerations,
-                    repositoryData.getGenId(),
-                    metadataForSnapshot(currentState.metadata(), request.includeGlobalState(), false, dataStreams, indexIds),
-                    snapshotInfo,
-                    version,
-                    state -> state,
-                    Priority.IMMEDIATE,
-                    new ActionListener<RepositoryData>() {
-                        @Override
-                        public void onResponse(RepositoryData repositoryData) {
-                            if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) {
-                                failSnapshotCompletionListeners(
-                                    snapshot,
-                                    new SnapshotException(snapshot, "Aborting snapshot-v2, no longer cluster manager")
-                                );
-                                listener.onFailure(
-                                    new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2, no longer cluster manager")
-                                );
-                                return;
+                pinnedTimestampListener.whenComplete(repoData -> {
+                    repository.finalizeSnapshot(
+                        shardGenerations,
+                        repositoryData.getGenId(),
+                        metadataForSnapshot(newState.metadata(), request.includeGlobalState(), false, dataStreams, newEntry.indices()),
+                        snapshotInfo,
+                        version,
+                        state -> stateWithoutSnapshot(state, snapshot),
+                        Priority.IMMEDIATE,
+                        new ActionListener<RepositoryData>() {
+                            @Override
+                            public void onResponse(RepositoryData repositoryData) {
+                                if (clusterService.state().nodes().isLocalNodeElectedClusterManager() == false) {
+                                    leaveRepoLoop(repositoryName);
+                                    failSnapshotCompletionListeners(
+                                        snapshot,
+                                        new SnapshotException(snapshot, "Aborting snapshot-v2, no longer cluster manager")
+                                    );
+                                    listener.onFailure(
+                                        new SnapshotException(
+                                            repositoryName,
+                                            snapshotName,
+                                            "Aborting snapshot-v2, no longer cluster manager"
+                                        )
+                                    );
+                                    return;
+                                }
+                                listener.onResponse(snapshotInfo);
+                                // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations
+                                // can get queued . This is triggering them.
+                                runNextQueuedOperation(repositoryData, repositoryName, true);
+                                cleanOrphanTimestamp(repositoryName, repositoryData);
                             }
-                            updateSnapshotPinnedTimestamp(repositoryData, snapshot, pinnedTimestamp, pinnedTimestampListener);
-                        }
 
-                        @Override
-                        public void onFailure(Exception e) {
-                            logger.error("Failed to upload files to snapshot repo {} for snapshot-v2 {} ", repositoryName, snapshotName);
-                            listener.onFailure(e);
+                            @Override
+                            public void onFailure(Exception e) {
+                                logger.error("Failed to finalize snapshot repo {} for snapshot-v2 {} ", repositoryName, snapshotName);
+                                leaveRepoLoop(repositoryName);
+                                // cleaning up in progress snapshot here
+                                stateWithoutSnapshotV2(newState);
+                                listener.onFailure(e);
+                            }
                         }
-                    }
-                );
+                    );
+                }, e -> {
+                    logger.error("Failed to update pinned timestamp for snapshot-v2 {} {} {} ", repositoryName, snapshotName, e);
+                    leaveRepoLoop(repositoryName);
+                    // cleaning up in progress snapshot here
+                    stateWithoutSnapshotV2(newState);
+                    listener.onFailure(e);
+                });
+                updateSnapshotPinnedTimestamp(repositoryData, snapshot, pinnedTimestamp, pinnedTimestampListener);
+            }
 
-            }, listener::onFailure);
-        } catch (Exception e) {
-            assert false : new AssertionError(e);
-            logger.error("Snapshot-v2 {} creation failed with exception {}", snapshot.getSnapshotId().getName(), e);
-            listener.onFailure(e);
+            @Override
+            public TimeValue timeout() {
+                return request.clusterManagerNodeTimeout();
+            }
+
+        }, "create_snapshot [" + snapshotName + ']', listener::onFailure);
+    }
+
+    private void cleanOrphanTimestamp(String repoName, RepositoryData repositoryData) {
+        Collection<String> snapshotUUIDs = repositoryData.getSnapshotIds().stream().map(SnapshotId::getUUID).collect(Collectors.toSet());
+        Map<String, List<Long>> pinnedEntities = RemoteStorePinnedTimestampService.getPinnedEntities();
+
+        List<String> orphanPinnedEntities = pinnedEntities.keySet()
+            .stream()
+            .filter(pinnedEntity -> isOrphanPinnedEntity(repoName, snapshotUUIDs, pinnedEntity))
+            .collect(Collectors.toList());
+
+        if (orphanPinnedEntities.isEmpty()) {
+            return;
+        }
+
+        logger.info("Found {} orphan timestamps. Cleaning it up now", orphanPinnedEntities.size());
+        if (tryEnterRepoLoop(repoName)) {
+            deleteOrphanTimestamps(pinnedEntities, orphanPinnedEntities);
+            leaveRepoLoop(repoName);
+        } else {
+            logger.info("Concurrent snapshot create/delete is happening. Skipping clean up of orphan timestamps");
+        }
+    }
+
+    private boolean isOrphanPinnedEntity(String repoName, Collection<String> snapshotUUIDs, String pinnedEntity) {
+        Tuple<String, String> tokens = getRepoSnapshotUUIDTuple(pinnedEntity);
+        return Objects.equals(tokens.v1(), repoName) && snapshotUUIDs.contains(tokens.v2()) == false;
+    }
+
+    private void deleteOrphanTimestamps(Map<String, List<Long>> pinnedEntities, List<String> orphanPinnedEntities) {
+        final CountDownLatch latch = new CountDownLatch(orphanPinnedEntities.size());
+        for (String pinnedEntity : orphanPinnedEntities) {
+            assert pinnedEntities.get(pinnedEntity).size() == 1 : "Multiple timestamps for same repo-snapshot uuid found";
+            long orphanTimestamp = pinnedEntities.get(pinnedEntity).get(0);
+            remoteStorePinnedTimestampService.unpinTimestamp(
+                orphanTimestamp,
+                pinnedEntity,
+                new LatchedActionListener<>(new ActionListener<>() {
+                    @Override
+                    public void onResponse(Void unused) {}
+
+                    @Override
+                    public void onFailure(Exception e) {}
+                }, latch)
+            );
+        }
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
         }
     }
 
@@ -613,7 +741,7 @@ private void updateSnapshotPinnedTimestamp(
     ) {
         remoteStorePinnedTimestampService.pinTimestamp(
             timestampToPin,
-            snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshot.getSnapshotId().getUUID(),
+            getPinningEntity(snapshot.getRepository(), snapshot.getSnapshotId().getUUID()),
             new ActionListener<Void>() {
                 @Override
                 public void onResponse(Void unused) {
@@ -631,6 +759,15 @@ public void onFailure(Exception e) {
         );
     }
 
+    public static String getPinningEntity(String repositoryName, String snapshotUUID) {
+        return repositoryName + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshotUUID;
+    }
+
+    public static Tuple<String, String> getRepoSnapshotUUIDTuple(String pinningEntity) {
+        String[] tokens = pinningEntity.split(SNAPSHOT_PINNED_TIMESTAMP_DELIMITER);
+        return new Tuple<>(tokens[0], tokens[1]);
+    }
+
     private void cloneSnapshotPinnedTimestamp(
         RepositoryData repositoryData,
         SnapshotId sourceSnapshot,
@@ -640,8 +777,8 @@ private void cloneSnapshotPinnedTimestamp(
     ) {
         remoteStorePinnedTimestampService.cloneTimestamp(
             timestampToPin,
-            snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + sourceSnapshot.getUUID(),
-            snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshot.getSnapshotId().getUUID(),
+            getPinningEntity(snapshot.getRepository(), sourceSnapshot.getUUID()),
+            getPinningEntity(snapshot.getRepository(), snapshot.getSnapshotId().getUUID()),
             new ActionListener<Void>() {
                 @Override
                 public void onResponse(Void unused) {
@@ -766,12 +903,24 @@ public void cloneSnapshotV2(
             private SnapshotId sourceSnapshotId;
             private List<String> indicesForSnapshot;
 
+            boolean enteredRepoLoop;
+
             @Override
             public ClusterState execute(ClusterState currentState) {
                 createSnapshotPreValidations(currentState, repositoryData, repositoryName, snapshotName);
                 final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
                 final List<SnapshotsInProgress.Entry> runningSnapshots = snapshots.entries();
 
+                // Entering finalize loop here to prevent concurrent snapshots v2 snapshots
+                enteredRepoLoop = tryEnterRepoLoop(repositoryName);
+                if (enteredRepoLoop == false) {
+                    throw new ConcurrentSnapshotExecutionException(
+                        repositoryName,
+                        snapshotName,
+                        "cannot start snapshot-v2 while a repository is in finalization state"
+                    );
+                }
+
                 sourceSnapshotId = repositoryData.getSnapshotIds()
                     .stream()
                     .filter(src -> src.getName().equals(request.source()))
@@ -795,14 +944,14 @@ public ClusterState execute(ClusterState currentState) {
                         indicesForSnapshot.add(indexId.getName());
                     }
                 }
-
                 newEntry = SnapshotsInProgress.startClone(
                     snapshot,
                     sourceSnapshotId,
                     repositoryData.resolveIndices(indicesForSnapshot),
                     threadPool.absoluteTimeInMillis(),
                     repositoryData.getGenId(),
-                    minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null)
+                    minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null),
+                    true
                 );
                 final List<SnapshotsInProgress.Entry> newEntries = new ArrayList<>(runningSnapshots);
                 newEntries.add(newEntry);
@@ -813,6 +962,9 @@ public ClusterState execute(ClusterState currentState) {
             public void onFailure(String source, Exception e) {
                 logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to clone snapshot-v2", repositoryName, snapshotName), e);
                 listener.onFailure(e);
+                if (enteredRepoLoop) {
+                    leaveRepoLoop(repositoryName);
+                }
             }
 
             @Override
@@ -839,67 +991,83 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl
                         true,
                         snapshotInfo.getPinnedTimestamp()
                     );
-                    if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) {
+                    if (clusterService.state().nodes().isLocalNodeElectedClusterManager() == false) {
                         throw new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2 clone, no longer cluster manager");
                     }
                     final StepListener<RepositoryData> pinnedTimestampListener = new StepListener<>();
                     pinnedTimestampListener.whenComplete(repoData -> {
-                        logger.info("snapshot-v2 clone [{}] completed successfully", snapshot);
-                        listener.onResponse(null);
-                    }, listener::onFailure);
-                    repository.finalizeSnapshot(
-                        shardGenerations,
-                        repositoryData.getGenId(),
-                        metadataForSnapshot(
-                            currentState.metadata(),
-                            newEntry.includeGlobalState(),
-                            false,
-                            newEntry.dataStreams(),
-                            newEntry.indices()
-                        ),
-                        cloneSnapshotInfo,
-                        repositoryData.getVersion(sourceSnapshotId),
-                        state -> stateWithoutSnapshot(state, snapshot),
-                        Priority.IMMEDIATE,
-                        new ActionListener<RepositoryData>() {
-                            @Override
-                            public void onResponse(RepositoryData repositoryData) {
-                                if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) {
-                                    failSnapshotCompletionListeners(
-                                        snapshot,
-                                        new SnapshotException(snapshot, "Aborting Snapshot-v2 clone, no longer cluster manager")
-                                    );
-                                    listener.onFailure(
-                                        new SnapshotException(
-                                            repositoryName,
-                                            snapshotName,
-                                            "Aborting Snapshot-v2 clone, no longer cluster manager"
-                                        )
-                                    );
-                                    return;
+                        repository.finalizeSnapshot(
+                            shardGenerations,
+                            repositoryData.getGenId(),
+                            metadataForSnapshot(
+                                currentState.metadata(),
+                                newEntry.includeGlobalState(),
+                                false,
+                                newEntry.dataStreams(),
+                                newEntry.indices()
+                            ),
+                            cloneSnapshotInfo,
+                            repositoryData.getVersion(sourceSnapshotId),
+                            state -> stateWithoutSnapshot(state, snapshot),
+                            Priority.IMMEDIATE,
+                            new ActionListener<RepositoryData>() {
+                                @Override
+                                public void onResponse(RepositoryData repositoryData) {
+                                    if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) {
+                                        leaveRepoLoop(repositoryName);
+                                        failSnapshotCompletionListeners(
+                                            snapshot,
+                                            new SnapshotException(snapshot, "Aborting Snapshot-v2 clone, no longer cluster manager")
+                                        );
+                                        listener.onFailure(
+                                            new SnapshotException(
+                                                repositoryName,
+                                                snapshotName,
+                                                "Aborting Snapshot-v2 clone, no longer cluster manager"
+                                            )
+                                        );
+                                        return;
+                                    }
+                                    logger.info("snapshot-v2 clone [{}] completed successfully", snapshot);
+                                    listener.onResponse(null);
+                                    // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations
+                                    // can get queued . This is triggering them.
+                                    runNextQueuedOperation(repositoryData, repositoryName, true);
                                 }
-                                cloneSnapshotPinnedTimestamp(
-                                    repositoryData,
-                                    sourceSnapshotId,
-                                    snapshot,
-                                    snapshotInfo.getPinnedTimestamp(),
-                                    pinnedTimestampListener
-                                );
-                            }
 
-                            @Override
-                            public void onFailure(Exception e) {
-                                logger.error(
-                                    "Failed to upload files to snapshot repo {} for clone snapshot-v2 {} ",
-                                    repositoryName,
-                                    snapshotName
-                                );
-                                listener.onFailure(e);
+                                @Override
+                                public void onFailure(Exception e) {
+                                    logger.error(
+                                        "Failed to upload files to snapshot repo {} for clone snapshot-v2 {} ",
+                                        repositoryName,
+                                        snapshotName
+                                    );
+                                    stateWithoutSnapshotV2(newState);
+                                    leaveRepoLoop(repositoryName);
+                                    listener.onFailure(e);
+                                }
                             }
-                        }
+                        );
+                    }, e -> {
+                        logger.error("Failed to update pinned timestamp for snapshot-v2 {} {} ", repositoryName, snapshotName);
+                        stateWithoutSnapshotV2(newState);
+                        leaveRepoLoop(repositoryName);
+                        listener.onFailure(e);
+                    });
+
+                    cloneSnapshotPinnedTimestamp(
+                        repositoryData,
+                        sourceSnapshotId,
+                        snapshot,
+                        snapshotInfo.getPinnedTimestamp(),
+                        pinnedTimestampListener
                     );
-
-                }, listener::onFailure);
+                }, e -> {
+                    logger.error("Failed to retrieve snapshot info for snapshot-v2 {} {} ", repositoryName, snapshotName);
+                    stateWithoutSnapshotV2(newState);
+                    leaveRepoLoop(repositoryName);
+                    listener.onFailure(e);
+                });
             }
 
             @Override
@@ -1482,6 +1650,13 @@ public void applyClusterState(ClusterChangedEvent event) {
                 // cluster-manager
                 SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
                 final boolean newClusterManager = event.previousState().nodes().isLocalNodeElectedClusterManager() == false;
+
+                if (newClusterManager && snapshotsInProgress.entries().isEmpty() == false) {
+                    // clean up snapshot v2 in progress or clone v2 present.
+                    // Snapshot v2 create and clone are sync operation . In case of cluster manager failures in midst , we won't
+                    // send ack to caller and won't continue on new cluster manager . Caller will need to retry it.
+                    stateWithoutSnapshotV2(event.state());
+                }
                 processExternalChanges(
                     newClusterManager || removedNodesCleanupNeeded(snapshotsInProgress, event.nodesDelta().removedNodes()),
                     event.routingTableChanged() && waitingShardsStartedOrUnassigned(snapshotsInProgress, event)
@@ -1571,7 +1746,7 @@ private static boolean assertNoDanglingSnapshots(ClusterState state) {
     /**
      * Updates the state of in-progress snapshots in reaction to a change in the configuration of the cluster nodes (cluster-manager fail-over or
      * disconnect of a data node that was executing a snapshot) or a routing change that started shards whose snapshot state is
-     * {@link SnapshotsInProgress.ShardState#WAITING}.
+     * {@link ShardState#WAITING}.
      *
      * @param changedNodes true iff either a cluster-manager fail-over occurred or a data node that was doing snapshot work got removed from the
      *                     cluster
@@ -1593,7 +1768,14 @@ private void processExternalChanges(boolean changedNodes, boolean startShards) {
                 @Override
                 public ClusterState execute(ClusterState currentState) {
                     RoutingTable routingTable = currentState.routingTable();
-                    final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
+                    SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
+                    // Removing shallow snapshots v2 as we we take care of these in stateWithoutSnapshotV2()
+                    snapshots = SnapshotsInProgress.of(
+                        snapshots.entries()
+                            .stream()
+                            .filter(snapshot -> snapshot.remoteStoreIndexShallowCopyV2() == false)
+                            .collect(Collectors.toList())
+                    );
                     DiscoveryNodes nodes = currentState.nodes();
                     boolean changed = false;
                     final EnumSet<State> statesToUpdate;
@@ -1650,7 +1832,7 @@ public ClusterState execute(ClusterState currentState) {
                             changed = true;
                             logger.debug("[{}] was found in dangling INIT or ABORTED state", snapshot);
                         } else {
-                            if (snapshot.state().completed() || completed(snapshot.shards().values())) {
+                            if ((snapshot.state().completed() || completed(snapshot.shards().values()))) {
                                 finishedSnapshots.add(snapshot);
                             }
                             updatedSnapshotEntries.add(snapshot);
@@ -2176,6 +2358,59 @@ private static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot sn
         return readyDeletions(result).v1();
     }
 
+    private void stateWithoutSnapshotV2(ClusterState state) {
+        SnapshotsInProgress snapshots = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
+        boolean changed = false;
+        ArrayList<SnapshotsInProgress.Entry> entries = new ArrayList<>();
+        for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
+            if (entry.remoteStoreIndexShallowCopyV2()) {
+                changed = true;
+            } else {
+                entries.add(entry);
+            }
+        }
+        if (changed) {
+            logger.info("Cleaning up in progress v2 snapshots now");
+            clusterService.submitStateUpdateTask(
+                "remove in progress snapshot v2 after cluster manager switch",
+                new ClusterStateUpdateTask() {
+                    @Override
+                    public ClusterState execute(ClusterState currentState) {
+                        SnapshotsInProgress snapshots = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
+                        boolean changed = false;
+                        ArrayList<SnapshotsInProgress.Entry> entries = new ArrayList<>();
+                        for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
+                            if (entry.remoteStoreIndexShallowCopyV2()) {
+                                changed = true;
+                            } else {
+                                entries.add(entry);
+                            }
+                        }
+                        if (changed) {
+                            return ClusterState.builder(currentState)
+                                .putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(unmodifiableList(entries)))
+                                .build();
+                        } else {
+                            return currentState;
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        // execute never fails , so we should never hit this.
+                        logger.warn(
+                            () -> new ParameterizedMessage(
+                                "failed to remove in progress snapshot v2 state after cluster manager switch {}",
+                                e
+                            ),
+                            e
+                        );
+                    }
+                }
+            );
+        }
+    }
+
     /**
      * Removes record of running snapshot from cluster state and notifies the listener when this action is complete. This method is only
      * used when the snapshot fails for some reason. During normal operation the snapshot repository will remove the
@@ -2416,7 +2651,7 @@ private static List<SnapshotId> matchingSnapshotIds(
                 }
             }
         }
-        return Collections.unmodifiableList(new ArrayList<>(foundSnapshots));
+        return unmodifiableList(new ArrayList<>(foundSnapshots));
     }
 
     // Return in-progress snapshot entries by name and repository in the given cluster state or null if none is found
@@ -2566,7 +2801,7 @@ public ClusterState execute(ClusterState currentState) {
                         reusedExistingDelete = true;
                         return currentState;
                     }
-                    final List<SnapshotId> toDelete = Collections.unmodifiableList(new ArrayList<>(snapshotIdsRequiringCleanup));
+                    final List<SnapshotId> toDelete = unmodifiableList(new ArrayList<>(snapshotIdsRequiringCleanup));
                     ensureBelowConcurrencyLimit(repoName, toDelete.get(0).getName(), snapshots, deletionsInProgress);
                     newDelete = new SnapshotDeletionsInProgress.Entry(
                         toDelete,
@@ -3124,7 +3359,7 @@ private static <T> void completeListenersIgnoringException(@Nullable List<Action
      * @param indices             Indices to snapshot
      * @return list of shard to be included into current snapshot
      */
-    private static Map<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards(
+    private static Map<ShardId, ShardSnapshotStatus> shards(
         SnapshotsInProgress snapshotsInProgress,
         @Nullable SnapshotDeletionsInProgress deletionsInProgress,
         Metadata metadata,
@@ -3133,7 +3368,7 @@ private static Map<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards(
         RepositoryData repositoryData,
         String repoName
     ) {
-        final Map<ShardId, SnapshotsInProgress.ShardSnapshotStatus> builder = new HashMap<>();
+        final Map<ShardId, ShardSnapshotStatus> builder = new HashMap<>();
         final ShardGenerations shardGenerations = repositoryData.shardGenerations();
         final InFlightShardSnapshotStates inFlightShardStates = InFlightShardSnapshotStates.forRepo(
             repoName,
@@ -3167,7 +3402,7 @@ private static Map<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards(
                     }
                     final ShardSnapshotStatus shardSnapshotStatus;
                     if (indexRoutingTable == null) {
-                        shardSnapshotStatus = new SnapshotsInProgress.ShardSnapshotStatus(
+                        shardSnapshotStatus = new ShardSnapshotStatus(
                             null,
                             ShardState.MISSING,
                             "missing routing table",
@@ -3326,6 +3561,9 @@ public boolean assertAllListenersResolved() {
             + " on ["
             + localNode
             + "]";
+        if (repositoryOperations.isEmpty() == false) {
+            logger.info("Not empty");
+        }
         assert repositoryOperations.isEmpty() : "Found leaked snapshots to finalize " + repositoryOperations + " on [" + localNode + "]";
         return true;
     }
diff --git a/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java b/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java
deleted file mode 100644
index 1689b3e4941ec..0000000000000
--- a/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-package org.opensearch.snapshots;
-
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.rest.RestStatus;
-
-import java.io.IOException;
-
-/**
- * Thrown if the number of shards across the requested resources (snapshot(s) or the index/indices of a particular snapshot)
- * breaches the limit of snapshot.max_shards_allowed_in_status_api cluster setting
- *
- * @opensearch.internal
- */
-public class TooManyShardsInSnapshotsStatusException extends SnapshotException {
-
-    public TooManyShardsInSnapshotsStatusException(
-        final String repositoryName,
-        final SnapshotId snapshotId,
-        final String message,
-        final Throwable cause
-    ) {
-        super(repositoryName, snapshotId, message, cause);
-    }
-
-    public TooManyShardsInSnapshotsStatusException(final String repositoryName, final String message, String... snapshotName) {
-        super(repositoryName, String.join(", ", snapshotName), message);
-    }
-
-    public TooManyShardsInSnapshotsStatusException(StreamInput in) throws IOException {
-        super(in);
-    }
-
-    @Override
-    public RestStatus status() {
-        return RestStatus.REQUEST_ENTITY_TOO_LARGE;
-    }
-}
diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java
index 343d4571593a7..2040703d88c38 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java
@@ -125,7 +125,7 @@ public void onTaskCompleted(Task task) {
         if (!TASKS_TO_TRACK.contains(task.getClass())) {
             return;
         }
-        this.cancelledTaskTracker.entrySet().removeIf(entry -> entry.getKey() == task.getId());
+        this.cancelledTaskTracker.remove(task.getId());
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
index 81220ab171b34..d795fd252b7fc 100644
--- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
@@ -53,6 +53,7 @@
 import org.opensearch.core.service.ReportingService;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.gateway.remote.ClusterStateChecksum;
 import org.opensearch.node.Node;
 
 import java.io.IOException;
@@ -118,6 +119,7 @@ public static class Names {
         public static final String REMOTE_RECOVERY = "remote_recovery";
         public static final String REMOTE_STATE_READ = "remote_state_read";
         public static final String INDEX_SEARCHER = "index_searcher";
+        public static final String REMOTE_STATE_CHECKSUM = "remote_state_checksum";
     }
 
     /**
@@ -191,6 +193,7 @@ public static ThreadPoolType fromType(String type) {
         map.put(Names.REMOTE_RECOVERY, ThreadPoolType.SCALING);
         map.put(Names.REMOTE_STATE_READ, ThreadPoolType.SCALING);
         map.put(Names.INDEX_SEARCHER, ThreadPoolType.RESIZABLE);
+        map.put(Names.REMOTE_STATE_CHECKSUM, ThreadPoolType.FIXED);
         THREAD_POOL_TYPES = Collections.unmodifiableMap(map);
     }
 
@@ -307,6 +310,10 @@ public ThreadPool(
                 runnableTaskListener
             )
         );
+        builders.put(
+            Names.REMOTE_STATE_CHECKSUM,
+            new FixedExecutorBuilder(settings, Names.REMOTE_STATE_CHECKSUM, ClusterStateChecksum.COMPONENT_SIZE, 1000)
+        );
 
         for (final ExecutorBuilder<?> builder : customBuilders) {
             if (builders.containsKey(builder.name())) {
diff --git a/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java b/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java
index e634323d58269..3a3e8c964b6c5 100644
--- a/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java
+++ b/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java
@@ -64,6 +64,15 @@ public class ClusterConnectionManager implements ConnectionManager {
 
     private final ConcurrentMap<DiscoveryNode, Transport.Connection> connectedNodes = ConcurrentCollections.newConcurrentMap();
     private final ConcurrentMap<DiscoveryNode, ListenableFuture<Void>> pendingConnections = ConcurrentCollections.newConcurrentMap();
+    /**
+     This set is used only by cluster-manager nodes.
+     Nodes are marked as pending disconnect right before cluster state publish phase.
+     They are cleared up as part of cluster state apply commit phase
+     This is to avoid connections from being made to nodes that are in the process of leaving the cluster
+     Note: If a disconnect is initiated while a connect is in progress, this Set will not handle this case.
+     Callers need to ensure that connects and disconnects are sequenced.
+     */
+    private final Set<DiscoveryNode> pendingDisconnections = ConcurrentCollections.newConcurrentSet();
     private final AbstractRefCounted connectingRefCounter = new AbstractRefCounted("connection manager") {
         @Override
         protected void closeInternal() {
@@ -122,12 +131,19 @@ public void connectToNode(
         ConnectionValidator connectionValidator,
         ActionListener<Void> listener
     ) throws ConnectTransportException {
+        logger.trace("connecting to node [{}]", node);
         ConnectionProfile resolvedProfile = ConnectionProfile.resolveConnectionProfile(connectionProfile, defaultProfile);
         if (node == null) {
             listener.onFailure(new ConnectTransportException(null, "can't connect to a null node"));
             return;
         }
 
+        // if node-left is still in progress, we fail the connect request early
+        if (pendingDisconnections.contains(node)) {
+            listener.onFailure(new IllegalStateException("cannot make a new connection as disconnect to node [" + node + "] is pending"));
+            return;
+        }
+
         if (connectingRefCounter.tryIncRef() == false) {
             listener.onFailure(new IllegalStateException("connection manager is closed"));
             return;
@@ -170,6 +186,7 @@ public void connectToNode(
                             conn.addCloseListener(ActionListener.wrap(() -> {
                                 logger.trace("unregistering {} after connection close and marking as disconnected", node);
                                 connectedNodes.remove(node, finalConnection);
+                                pendingDisconnections.remove(node);
                                 connectionListener.onNodeDisconnected(node, conn);
                             }));
                         }
@@ -226,6 +243,19 @@ public void disconnectFromNode(DiscoveryNode node) {
             // if we found it and removed it we close
             nodeChannels.close();
         }
+        pendingDisconnections.remove(node);
+        logger.trace("Removed node [{}] from pending disconnections list", node);
+    }
+
+    @Override
+    public void setPendingDisconnection(DiscoveryNode node) {
+        logger.trace("marking disconnection as pending for node: [{}]", node);
+        pendingDisconnections.add(node);
+    }
+
+    @Override
+    public void clearPendingDisconnections() {
+        pendingDisconnections.clear();
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/transport/ConnectionManager.java b/server/src/main/java/org/opensearch/transport/ConnectionManager.java
index 10cfc2907098f..ebd5ccf29c8cc 100644
--- a/server/src/main/java/org/opensearch/transport/ConnectionManager.java
+++ b/server/src/main/java/org/opensearch/transport/ConnectionManager.java
@@ -65,6 +65,10 @@ void connectToNode(
 
     void disconnectFromNode(DiscoveryNode node);
 
+    void setPendingDisconnection(DiscoveryNode node);
+
+    void clearPendingDisconnections();
+
     Set<DiscoveryNode> getAllConnectedNodes();
 
     int size();
diff --git a/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java b/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java
index bd646f10df517..52f29bea8050d 100644
--- a/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java
+++ b/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java
@@ -114,6 +114,16 @@ public void disconnectFromNode(DiscoveryNode node) {
         delegate.disconnectFromNode(node);
     }
 
+    @Override
+    public void setPendingDisconnection(DiscoveryNode node) {
+        delegate.setPendingDisconnection(node);
+    }
+
+    @Override
+    public void clearPendingDisconnections() {
+        delegate.clearPendingDisconnections();
+    }
+
     @Override
     public ConnectionProfile getConnectionProfile() {
         return delegate.getConnectionProfile();
diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java
index fff6d82b23c7e..fe8631aa5ca3d 100644
--- a/server/src/main/java/org/opensearch/transport/TransportService.java
+++ b/server/src/main/java/org/opensearch/transport/TransportService.java
@@ -773,6 +773,18 @@ public void disconnectFromNode(DiscoveryNode node) {
         connectionManager.disconnectFromNode(node);
     }
 
+    public void setPendingDisconnection(DiscoveryNode node) {
+        connectionManager.setPendingDisconnection(node);
+    }
+
+    /**
+     * Wipes out all pending disconnections.
+     * This is called on cluster-manager failover to remove stale entries
+     */
+    public void clearPendingDisconnections() {
+        connectionManager.clearPendingDisconnections();
+    }
+
     public void addMessageListener(TransportMessageListener listener) {
         messageListener.listeners.add(listener);
     }
diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java
index eff312a36dbc6..c1972daeab6d3 100644
--- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java
@@ -65,6 +65,7 @@
 import org.opensearch.cluster.routing.UnsupportedWeightedRoutingStateException;
 import org.opensearch.cluster.service.ClusterManagerThrottlingException;
 import org.opensearch.common.UUIDs;
+import org.opensearch.common.breaker.ResponseLimitBreachedException;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.io.stream.BytesStreamOutput;
@@ -119,7 +120,6 @@
 import org.opensearch.snapshots.SnapshotId;
 import org.opensearch.snapshots.SnapshotInProgressException;
 import org.opensearch.snapshots.SnapshotInUseDeletionException;
-import org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.VersionUtils;
 import org.opensearch.transport.ActionNotFoundTransportException;
@@ -899,7 +899,7 @@ public void testIds() {
         ids.put(172, ViewNotFoundException.class);
         ids.put(173, ViewAlreadyExistsException.class);
         ids.put(174, InvalidIndexContextException.class);
-        ids.put(175, TooManyShardsInSnapshotsStatusException.class);
+        ids.put(175, ResponseLimitBreachedException.class);
         ids.put(10001, IndexCreateBlockException.class);
 
         Map<Class<? extends OpenSearchException>, Integer> reverse = new HashMap<>();
diff --git a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java
index c7e1039686cc9..26c4670b4288c 100644
--- a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java
+++ b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java
@@ -11,7 +11,9 @@
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
 import org.opensearch.client.node.NodeClient;
+import org.opensearch.common.breaker.ResponseLimitSettings;
 import org.opensearch.common.logging.DeprecationLogger;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsFilter;
 import org.opensearch.core.common.bytes.BytesArray;
@@ -155,7 +157,10 @@ public void testCatAllocation() {
     }
 
     public void testCatIndices() {
-        RestIndicesAction action = new RestIndicesAction();
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        Settings settings = Settings.builder().build();
+        ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings);
+        RestIndicesAction action = new RestIndicesAction(responseLimitSettings);
         Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(getRestRequestWithBothParams(), client));
         assertThat(e.getMessage(), containsString(DUPLICATE_PARAMETER_ERROR_MESSAGE));
         assertWarnings(MASTER_TIMEOUT_DEPRECATED_MESSAGE);
@@ -239,7 +244,10 @@ public void testCatThreadPool() {
     }
 
     public void testCatSegments() {
-        RestSegmentsAction action = new RestSegmentsAction();
+        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final Settings settings = Settings.builder().build();
+        final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings);
+        RestSegmentsAction action = new RestSegmentsAction(responseLimitSettings);
         Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(getRestRequestWithBothParams(), client));
         assertThat(e.getMessage(), containsString(DUPLICATE_PARAMETER_ERROR_MESSAGE));
         assertWarnings(MASTER_TIMEOUT_DEPRECATED_MESSAGE);
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
index 11902728eed07..34065daff2b8a 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -95,6 +95,7 @@
 import org.opensearch.node.NodeResourceUsageStats;
 import org.opensearch.node.NodesResourceUsageStats;
 import org.opensearch.node.ResponseCollectorService;
+import org.opensearch.node.remotestore.RemoteStoreNodeStats;
 import org.opensearch.ratelimitting.admissioncontrol.controllers.AdmissionController;
 import org.opensearch.ratelimitting.admissioncontrol.controllers.CpuBasedAdmissionController;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType;
@@ -614,6 +615,14 @@ public void testSerialization() throws IOException {
                 } else {
                     assertEquals(nodeCacheStats, deserializedNodeCacheStats);
                 }
+
+                RemoteStoreNodeStats remoteStoreNodeStats = nodeStats.getRemoteStoreNodeStats();
+                RemoteStoreNodeStats deserializedRemoteStoreNodeStats = deserializedNodeStats.getRemoteStoreNodeStats();
+                if (remoteStoreNodeStats == null) {
+                    assertNull(deserializedRemoteStoreNodeStats);
+                } else {
+                    assertEquals(remoteStoreNodeStats, deserializedRemoteStoreNodeStats);
+                }
             }
         }
     }
@@ -996,6 +1005,16 @@ public void apply(String action, AdmissionControlActionType admissionControlActi
             nodeCacheStats = new NodeCacheStats(cacheStatsMap, flags);
         }
 
+        RemoteStoreNodeStats remoteStoreNodeStats = null;
+        if (frequently()) {
+            remoteStoreNodeStats = new RemoteStoreNodeStats() {
+                @Override
+                public long getLastSuccessfulFetchOfPinnedTimestamps() {
+                    return 123456L;
+                }
+            };
+        }
+
         // TODO: Only remote_store based aspects of NodeIndicesStats are being tested here.
         // It is possible to test other metrics in NodeIndicesStats as well since it extends Writeable now
         return new NodeStats(
@@ -1027,7 +1046,8 @@ public void apply(String action, AdmissionControlActionType admissionControlActi
             segmentReplicationRejectionStats,
             null,
             admissionControlStats,
-            nodeCacheStats
+            nodeCacheStats,
+            remoteStoreNodeStats
         );
     }
 
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java
index 1c4a77905d73f..823661ba14abf 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java
@@ -349,6 +349,7 @@ private ClusterStatsNodeResponse createClusterStatsNodeResponse(
             null,
             null,
             null,
+            null,
             null
         );
         if (defaultBehavior) {
diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
index 40514c526f190..acda1445bacbb 100644
--- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
@@ -42,6 +42,8 @@
 import org.opensearch.geometry.LinearRing;
 import org.opensearch.index.query.GeoShapeQueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.rest.action.search.RestSearchAction;
 import org.opensearch.search.AbstractSearchTestCase;
 import org.opensearch.search.Scroll;
 import org.opensearch.search.builder.PointInTimeBuilder;
@@ -50,14 +52,18 @@
 import org.opensearch.search.rescore.QueryRescorerBuilder;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.VersionUtils;
+import org.opensearch.test.rest.FakeRestRequest;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.IntConsumer;
 
 import static java.util.Collections.emptyMap;
+import static org.opensearch.action.search.SearchType.DFS_QUERY_THEN_FETCH;
 import static org.opensearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
 
 public class SearchRequestTests extends AbstractSearchTestCase {
 
@@ -242,6 +248,19 @@ public void testCopyConstructor() throws IOException {
         assertNotSame(deserializedRequest, searchRequest);
     }
 
+    public void testParseSearchRequestWithUnsupportedSearchType() throws IOException {
+        RestRequest restRequest = new FakeRestRequest();
+        SearchRequest searchRequest = createSearchRequest();
+        IntConsumer setSize = mock(IntConsumer.class);
+        restRequest.params().put("search_type", "query_and_fetch");
+
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> RestSearchAction.parseSearchRequest(searchRequest, restRequest, null, namedWriteableRegistry, setSize)
+        );
+        assertEquals("Unsupported search type [query_and_fetch]", exception.getMessage());
+    }
+
     public void testEqualsAndHashcode() throws IOException {
         checkEqualsAndHashCode(createSearchRequest(), SearchRequest::new, this::mutate);
     }
@@ -268,10 +287,7 @@ private SearchRequest mutate(SearchRequest searchRequest) {
         );
         mutators.add(
             () -> mutation.searchType(
-                randomValueOtherThan(
-                    searchRequest.searchType(),
-                    () -> randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)
-                )
+                randomValueOtherThan(searchRequest.searchType(), () -> randomFrom(DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH))
             )
         );
         mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder)));
diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
index 5539dd26dd52d..cd050fb346563 100644
--- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
+++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
@@ -195,6 +195,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -226,6 +227,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -257,6 +259,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
@@ -319,6 +322,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -350,6 +354,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -381,6 +386,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
diff --git a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java
index 4cf82f1dabab3..4500860c937ea 100644
--- a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java
@@ -35,6 +35,9 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
 import org.opensearch.OpenSearchTimeoutException;
 import org.opensearch.Version;
 import org.opensearch.action.support.PlainActionFuture;
@@ -53,9 +56,11 @@
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.MockLogAppender;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.test.TestLogsAppender;
 import org.opensearch.test.junit.annotations.TestLogging;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.ClusterConnectionManager;
 import org.opensearch.transport.ConnectTransportException;
 import org.opensearch.transport.ConnectionProfile;
 import org.opensearch.transport.Transport;
@@ -69,6 +74,7 @@
 import org.junit.Before;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -77,6 +83,7 @@
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
 
 import static java.util.Collections.emptySet;
@@ -86,12 +93,15 @@
 import static org.opensearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
 import static org.opensearch.node.Node.NODE_NAME_SETTING;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 
 public class NodeConnectionsServiceTests extends OpenSearchTestCase {
 
     private ThreadPool threadPool;
     private TransportService transportService;
     private Map<DiscoveryNode, CheckedRunnable<Exception>> nodeConnectionBlocks;
+    private TestLogsAppender testLogsAppender;
+    private LoggerContext loggerContext;
 
     private List<DiscoveryNode> generateNodes() {
         List<DiscoveryNode> nodes = new ArrayList<>();
@@ -490,6 +500,108 @@ public void testDebugLogging() throws IllegalAccessException {
         }
     }
 
+    public void testConnectionCheckerRetriesIfPendingDisconnection() throws InterruptedException {
+        final Settings.Builder settings = Settings.builder();
+        final long reconnectIntervalMillis = 50;
+        settings.put(CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), reconnectIntervalMillis + "ms");
+
+        final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(
+            builder().put(NODE_NAME_SETTING.getKey(), "node").build(),
+            random()
+        );
+
+        MockTransport transport = new MockTransport(deterministicTaskQueue.getThreadPool());
+        TestTransportService transportService = new TestTransportService(transport, deterministicTaskQueue.getThreadPool());
+        transportService.start();
+        transportService.acceptIncomingRequests();
+
+        final TestNodeConnectionsService service = new TestNodeConnectionsService(
+            settings.build(),
+            deterministicTaskQueue.getThreadPool(),
+            transportService
+        );
+        service.start();
+
+        // setup the connections
+        final DiscoveryNode node = new DiscoveryNode("node0", buildNewFakeTransportAddress(), Version.CURRENT);
+
+        final DiscoveryNodes nodes = DiscoveryNodes.builder().add(node).build();
+
+        final AtomicBoolean connectionCompleted = new AtomicBoolean();
+        service.connectToNodes(nodes, () -> connectionCompleted.set(true));
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertTrue(connectionCompleted.get());
+
+        // reset any logs as we want to assert for exceptions that show up after this
+        // reset connect to node count to assert for later
+        logger.info("--> resetting captured logs and counters");
+        testLogsAppender.clearCapturedLogs();
+        // this ensures we only track connection attempts that happen after the disconnection
+        transportService.resetConnectToNodeCallCount();
+
+        // block connection checker reconnection attempts until after we set pending disconnections
+        logger.info("--> disabling connection checker, and triggering disconnect");
+        service.setShouldReconnect(false);
+        transportService.disconnectFromNode(node);
+
+        // set pending disconnections to true to fail future reconnection attempts
+        final long maxDisconnectionTime = 1000;
+        deterministicTaskQueue.scheduleNow(new Runnable() {
+            @Override
+            public void run() {
+                logger.info("--> setting pending disconnections to fail next connection attempts");
+                service.setPendingDisconnections(new HashSet<>(Collections.singleton(node)));
+            }
+
+            @Override
+            public String toString() {
+                return "scheduled disconnection of " + node;
+            }
+        });
+        // our task queue will have the first task as the runnable to set pending disconnections
+        // here we re-enable the connection checker to enqueue next tasks for attempting reconnection
+        logger.info("--> re-enabling reconnection checker");
+        service.setShouldReconnect(true);
+
+        final long maxReconnectionTime = 2000;
+        final int expectedReconnectionAttempts = 10;
+
+        // this will first run the task to set the pending disconnections, then will execute the reconnection tasks
+        // exit early when we have enough reconnection attempts
+        logger.info("--> running tasks in order until expected reconnection attempts");
+        runTasksInOrderUntilExpectedReconnectionAttempts(
+            deterministicTaskQueue,
+            maxDisconnectionTime + maxReconnectionTime,
+            transportService,
+            expectedReconnectionAttempts
+        );
+        logger.info("--> verifying that connectionchecker tried to reconnect");
+
+        // assert that the connections failed
+        assertFalse("connected to " + node, transportService.nodeConnected(node));
+
+        // assert that we saw at least the required number of reconnection attempts, and the exceptions that showed up are as expected
+        logger.info("--> number of reconnection attempts: {}", transportService.getConnectToNodeCallCount());
+        assertThat(
+            "Did not see enough reconnection attempts from connection checker",
+            transportService.getConnectToNodeCallCount(),
+            greaterThan(expectedReconnectionAttempts)
+        );
+        boolean logFound = testLogsAppender.waitForLog("failed to connect", 1, TimeUnit.SECONDS)
+            && testLogsAppender.waitForLog(
+                "IllegalStateException: cannot make a new connection as disconnect to node",
+                1,
+                TimeUnit.SECONDS
+            );
+        assertTrue("Expected log for reconnection failure was not found in the required time period", logFound);
+
+        // clear the pending disconnections and ensure the connection gets re-established automatically by connectionchecker
+        logger.info("--> clearing pending disconnections to allow connections to re-establish");
+        service.clearPendingDisconnections();
+        runTasksUntil(deterministicTaskQueue, maxDisconnectionTime + maxReconnectionTime + 2 * reconnectIntervalMillis);
+        assertConnectedExactlyToNodes(transportService, nodes);
+    }
+
     private void runTasksUntil(DeterministicTaskQueue deterministicTaskQueue, long endTimeMillis) {
         while (deterministicTaskQueue.getCurrentTimeMillis() < endTimeMillis) {
             if (deterministicTaskQueue.hasRunnableTasks() && randomBoolean()) {
@@ -501,6 +613,24 @@ private void runTasksUntil(DeterministicTaskQueue deterministicTaskQueue, long e
         deterministicTaskQueue.runAllRunnableTasks();
     }
 
+    private void runTasksInOrderUntilExpectedReconnectionAttempts(
+        DeterministicTaskQueue deterministicTaskQueue,
+        long endTimeMillis,
+        TestTransportService transportService,
+        int expectedReconnectionAttempts
+    ) {
+        // break the loop if we timeout or if we have enough reconnection attempts
+        while ((deterministicTaskQueue.getCurrentTimeMillis() < endTimeMillis)
+            && (transportService.getConnectToNodeCallCount() <= expectedReconnectionAttempts)) {
+            if (deterministicTaskQueue.hasRunnableTasks() && randomBoolean()) {
+                deterministicTaskQueue.runNextTask();
+            } else if (deterministicTaskQueue.hasDeferredTasks()) {
+                deterministicTaskQueue.advanceTime();
+            }
+        }
+        deterministicTaskQueue.runAllRunnableTasksInEnqueuedOrder();
+    }
+
     private void ensureConnections(NodeConnectionsService service) {
         final PlainActionFuture<Void> future = new PlainActionFuture<>();
         service.ensureConnections(() -> future.onResponse(null));
@@ -526,6 +656,16 @@ private void assertConnected(TransportService transportService, Iterable<Discove
     @Before
     public void setUp() throws Exception {
         super.setUp();
+        // Add any other specific messages you want to capture
+        List<String> messagesToCapture = Arrays.asList("failed to connect", "IllegalStateException");
+        testLogsAppender = new TestLogsAppender(messagesToCapture);
+        loggerContext = (LoggerContext) LogManager.getContext(false);
+        Configuration config = loggerContext.getConfiguration();
+        LoggerConfig loggerConfig = config.getLoggerConfig(NodeConnectionsService.class.getName());
+        loggerConfig.addAppender(testLogsAppender, null, null);
+        loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName());
+        loggerConfig.addAppender(testLogsAppender, null, null);
+        loggerContext.updateLoggers();
         ThreadPool threadPool = new TestThreadPool(getClass().getName());
         this.threadPool = threadPool;
         nodeConnectionBlocks = newConcurrentMap();
@@ -537,6 +677,14 @@ public void setUp() throws Exception {
     @Override
     @After
     public void tearDown() throws Exception {
+        testLogsAppender.clearCapturedLogs();
+        loggerContext = (LoggerContext) LogManager.getContext(false);
+        Configuration config = loggerContext.getConfiguration();
+        LoggerConfig loggerConfig = config.getLoggerConfig(NodeConnectionsService.class.getName());
+        loggerConfig.removeAppender(testLogsAppender.getName());
+        loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName());
+        loggerConfig.removeAppender(testLogsAppender.getName());
+        loggerContext.updateLoggers();
         transportService.stop();
         ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
         threadPool = null;
@@ -545,6 +693,8 @@ public void tearDown() throws Exception {
 
     private final class TestTransportService extends TransportService {
 
+        private final AtomicInteger connectToNodeCallCount = new AtomicInteger(0);
+
         private TestTransportService(Transport transport, ThreadPool threadPool) {
             super(
                 Settings.EMPTY,
@@ -588,6 +738,47 @@ public void connectToNode(DiscoveryNode node, ActionListener<Void> listener) thr
             } else {
                 super.connectToNode(node, listener);
             }
+            logger.info("calling connectToNode");
+            connectToNodeCallCount.incrementAndGet();
+        }
+
+        public int getConnectToNodeCallCount() {
+            return connectToNodeCallCount.get();
+        }
+
+        public void resetConnectToNodeCallCount() {
+            connectToNodeCallCount.set(0);
+        }
+    }
+
+    private class TestNodeConnectionsService extends NodeConnectionsService {
+        private boolean shouldReconnect = true;
+
+        public TestNodeConnectionsService(Settings settings, ThreadPool threadPool, TransportService transportService) {
+            super(settings, threadPool, transportService);
+        }
+
+        public void setShouldReconnect(boolean shouldReconnect) {
+            this.shouldReconnect = shouldReconnect;
+        }
+
+        @Override
+        protected void doStart() {
+            final StoppableConnectionChecker connectionChecker = new StoppableConnectionChecker();
+            this.connectionChecker = connectionChecker;
+            connectionChecker.scheduleNextCheck();
+        }
+
+        class StoppableConnectionChecker extends NodeConnectionsService.ConnectionChecker {
+            @Override
+            protected void doRun() {
+                if (connectionChecker == this && shouldReconnect) {
+                    connectDisconnectedTargets(this::scheduleNextCheck);
+                } else {
+                    // Skip reconnection attempt but still schedule the next check
+                    scheduleNextCheck();
+                }
+            }
         }
     }
 
diff --git a/server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java b/server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java
new file mode 100644
index 0000000000000..4633d055b3505
--- /dev/null
+++ b/server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java
@@ -0,0 +1,228 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.breaker;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.routing.IndexRoutingTable;
+import org.opensearch.cluster.routing.IndexShardRoutingTable;
+import org.opensearch.cluster.routing.RoutingTable;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardRoutingState;
+import org.opensearch.cluster.routing.TestShardRouting;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.index.Index;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.opensearch.common.breaker.ResponseLimitSettings.CAT_INDICES_RESPONSE_LIMIT_SETTING;
+import static org.opensearch.common.breaker.ResponseLimitSettings.CAT_SEGMENTS_RESPONSE_LIMIT_SETTING;
+import static org.opensearch.common.breaker.ResponseLimitSettings.CAT_SHARDS_RESPONSE_LIMIT_SETTING;
+
+public class ResponseLimitSettingsTests extends OpenSearchTestCase {
+
+    public void testIsResponseLimitBreachedForNullMetadataExpectNotBreached() {
+        final Settings settings = Settings.builder().build();
+        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings);
+        final boolean breached = ResponseLimitSettings.isResponseLimitBreached(
+            (Metadata) null,
+            ResponseLimitSettings.LimitEntity.INDICES,
+            0
+        );
+        assertFalse(breached);
+    }
+
+    public void testIsResponseLimitBreachedForNullRoutingTableExpectNotBreached() {
+        final Settings settings = Settings.builder().build();
+        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings);
+        final boolean breached = ResponseLimitSettings.isResponseLimitBreached(
+            (RoutingTable) null,
+            ResponseLimitSettings.LimitEntity.INDICES,
+            0
+        );
+        assertFalse(breached);
+    }
+
+    public void testIsResponseLimitBreachedForNullLimitEntityWithRoutingTableExpectException() {
+        final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3");
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> ResponseLimitSettings.isResponseLimitBreached(clusterState.getRoutingTable(), null, 4)
+        );
+    }
+
+    public void testIsResponseLimitBreachedForNullLimitEntityWithMetadataExpectException() {
+        final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3");
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> ResponseLimitSettings.isResponseLimitBreached(clusterState.getMetadata(), null, 4)
+        );
+    }
+
+    public void testIsResponseLimitBreachedForCatIndicesWithNoSettingPassedExpectNotBreached() {
+        // Don't set setting
+        final Settings settings = Settings.builder().build();
+        final boolean breached = isBreachedForIndices(settings);
+        assertFalse(breached);
+    }
+
+    public void testIsResponseLimitBreachedForCatIndicesWithSettingDisabledExpectNotBreached() {
+        // Don't enable limit
+        final Settings settings = Settings.builder().put(CAT_INDICES_RESPONSE_LIMIT_SETTING.getKey(), -1).build();
+        final boolean breached = isBreachedForIndices(settings);
+        assertFalse(breached);
+    }
+
+    public void testIsResponseLimitBreachedForCatIndicesWithSettingEnabledExpectBreached() {
+        // Set limit of 1 index
+        final Settings settings = Settings.builder().put(CAT_INDICES_RESPONSE_LIMIT_SETTING.getKey(), 1).build();
+        final boolean breached = isBreachedForIndices(settings);
+        assertTrue(breached);
+    }
+
+    public void testIsResponseLimitBreachedForCatIndicesWithSettingEnabledExpectNotBreached() {
+        // Set limit of 5 indices
+        final Settings settings = Settings.builder().put(CAT_INDICES_RESPONSE_LIMIT_SETTING.getKey(), 5).build();
+        final boolean breached = isBreachedForIndices(settings);
+        assertFalse(breached);
+    }
+
+    private static boolean isBreachedForIndices(final Settings settings) {
+        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings);
+        // Pass cluster state with 3 indices
+        final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3");
+        return ResponseLimitSettings.isResponseLimitBreached(
+            clusterState.getMetadata(),
+            ResponseLimitSettings.LimitEntity.INDICES,
+            responseLimitSettings.getCatIndicesResponseLimit()
+        );
+    }
+
+    public void testIsResponseLimitBreachedForCatShardsWithSettingDisabledExpectNotBreached() {
+        // Don't enable limit
+        final Settings settings = Settings.builder().put(CAT_SHARDS_RESPONSE_LIMIT_SETTING.getKey(), -1).build();
+        final boolean breached = isBreachedForShards(settings);
+        assertFalse(breached);
+    }
+
+    public void testIsResponseLimitBreachedForCatShardsWithNoSettingPassedExpectNotBreached() {
+        // Don't set setting
+        final Settings settings = Settings.builder().build();
+        final boolean breached = isBreachedForShards(settings);
+        assertFalse(breached);
+    }
+
+    private static boolean isBreachedForShards(Settings settings) {
+        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings);
+        // Build cluster state with 3 shards
+        final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3");
+        return ResponseLimitSettings.isResponseLimitBreached(
+            clusterState.getRoutingTable(),
+            ResponseLimitSettings.LimitEntity.SHARDS,
+            responseLimitSettings.getCatShardsResponseLimit()
+        );
+    }
+
+    public void testIsResponseLimitBreachedForCatShardsWithSettingEnabledExpectBreached() {
+        // Set limit of 2 shards
+        final Settings settings = Settings.builder().put(CAT_SHARDS_RESPONSE_LIMIT_SETTING.getKey(), 2).build();
+        final boolean breached = isBreachedForShards(settings);
+        assertTrue(breached);
+    }
+
+    public void testIsResponseLimitBreachedForCatShardsWithSettingEnabledExpectNotBreached() {
+        // Set limit of 9 shards
+        final Settings settings = Settings.builder().put(CAT_SHARDS_RESPONSE_LIMIT_SETTING.getKey(), 9).build();
+        final boolean breached = isBreachedForShards(settings);
+        assertFalse(breached);
+    }
+
+    public void testIsResponseLimitBreachedForCatSegmentsWithSettingDisabledExpectNotBreached() {
+        // Don't enable limit
+        final Settings settings = Settings.builder().put(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.getKey(), -1).build();
+        final boolean breached = isBreachedForSegments(settings);
+        assertFalse(breached);
+    }
+
+    public void testIsResponseLimitBreachedForCatSegmentsWithNoSettingPassedExpectNotBreached() {
+        // Don't set setting
+        final Settings settings = Settings.builder().build();
+        final boolean breached = isBreachedForSegments(settings);
+        assertFalse(breached);
+    }
+
+    private static boolean isBreachedForSegments(Settings settings) {
+        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings);
+        // Build cluster state with 3 indices
+        final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3");
+        final boolean breached = ResponseLimitSettings.isResponseLimitBreached(
+            clusterState.getRoutingTable(),
+            ResponseLimitSettings.LimitEntity.INDICES,
+            responseLimitSettings.getCatSegmentsResponseLimit()
+        );
+        return breached;
+    }
+
+    public void testIsResponseLimitBreachedForCatSegmentsWithSettingEnabledExpectBreached() {
+        // Set limit of 1 index
+        final Settings settings = Settings.builder().put(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.getKey(), 1).build();
+        final boolean breached = isBreachedForSegments(settings);
+        assertTrue(breached);
+    }
+
+    public void testIsResponseLimitBreachedForCatSegmentsWithSettingEnabledExpectNotBreached() {
+        // Set limit of 5 indices
+        final Settings settings = Settings.builder().put(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.getKey(), 5).build();
+        final boolean breached = isBreachedForSegments(settings);
+        assertFalse(breached);
+    }
+
+    private static ClusterState buildClusterState(String... indices) {
+        final Metadata.Builder metadata = Metadata.builder();
+        for (String index : indices) {
+            metadata.put(IndexMetadata.builder(index).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0));
+        }
+        final Map<String, IndexRoutingTable> indexRoutingTableMap = new HashMap<>();
+        for (String s : indices) {
+            final Index index = new Index(s, "uuid");
+            final ShardId primaryShardId = new ShardId(index, 0);
+            final ShardRouting primaryShardRouting = createShardRouting(primaryShardId, true);
+            final ShardId replicaShardId = new ShardId(index, 1);
+            final ShardRouting replicaShardRouting = createShardRouting(replicaShardId, false);
+            final IndexShardRoutingTable.Builder indexShardRoutingTableBuilder = new IndexShardRoutingTable.Builder(primaryShardId);
+            indexShardRoutingTableBuilder.addShard(primaryShardRouting);
+            indexShardRoutingTableBuilder.addShard(replicaShardRouting);
+            final IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+                .addShard(primaryShardRouting)
+                .addShard(replicaShardRouting)
+                .addIndexShard(indexShardRoutingTableBuilder.build());
+            indexRoutingTableMap.put(index.getName(), indexRoutingTable.build());
+        }
+        final RoutingTable routingTable = new RoutingTable(1, indexRoutingTableMap);
+        return ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metadata(metadata)
+            .routingTable(routingTable)
+            .build();
+    }
+
+    private static ShardRouting createShardRouting(ShardId shardId, boolean isPrimary) {
+        return TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(4), isPrimary, ShardRoutingState.STARTED);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/common/settings/SettingTests.java b/server/src/test/java/org/opensearch/common/settings/SettingTests.java
index 7ebee680e8e52..c3c399a9d88b2 100644
--- a/server/src/test/java/org/opensearch/common/settings/SettingTests.java
+++ b/server/src/test/java/org/opensearch/common/settings/SettingTests.java
@@ -1274,6 +1274,20 @@ public void testFloatParser() throws Exception {
     public void testDoubleWithDefaultValue() {
         Setting<Double> doubleSetting = Setting.doubleSetting("foo.bar", 42.1);
         assertEquals(doubleSetting.get(Settings.EMPTY), Double.valueOf(42.1));
+
+        Setting<Double> doubleSettingWithValidator = Setting.doubleSetting("foo.bar", 42.1, value -> {
+            if (value <= 0.0) {
+                throw new IllegalArgumentException("The setting foo.bar must be >0");
+            }
+        });
+        try {
+            assertThrows(
+                IllegalArgumentException.class,
+                () -> doubleSettingWithValidator.get(Settings.builder().put("foo.bar", randomFrom(-1, 0)).build())
+            );
+        } catch (IllegalArgumentException ex) {
+            assertEquals("The setting foo.bar must be >0", ex.getMessage());
+        }
     }
 
     public void testDoubleWithFallbackValue() {
@@ -1282,6 +1296,20 @@ public void testDoubleWithFallbackValue() {
         assertEquals(doubleSetting.get(Settings.EMPTY), Double.valueOf(2.1));
         assertEquals(doubleSetting.get(Settings.builder().put("foo.bar", 3.2).build()), Double.valueOf(3.2));
         assertEquals(doubleSetting.get(Settings.builder().put("foo.baz", 3.2).build()), Double.valueOf(3.2));
+
+        Setting<Double> doubleSettingWithValidator = Setting.doubleSetting("foo.bar", fallbackSetting, value -> {
+            if (value <= 0.0) {
+                throw new IllegalArgumentException("The setting foo.bar must be >0");
+            }
+        });
+        try {
+            assertThrows(
+                IllegalArgumentException.class,
+                () -> doubleSettingWithValidator.get(Settings.builder().put("foo.bar", randomFrom(-1, 0)).build())
+            );
+        } catch (IllegalArgumentException ex) {
+            assertEquals("The setting foo.bar must be >0", ex.getMessage());
+        }
     }
 
     public void testDoubleWithMinMax() throws Exception {
diff --git a/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java b/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java
index 6b6c7b96266d3..6d9d1aad3c5d5 100644
--- a/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java
+++ b/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java
@@ -14,7 +14,6 @@
 
 import static org.opensearch.common.util.FeatureFlags.DATETIME_FORMATTER_CACHING;
 import static org.opensearch.common.util.FeatureFlags.EXTENSIONS;
-import static org.opensearch.common.util.FeatureFlags.IDENTITY;
 
 public class FeatureFlagTests extends OpenSearchTestCase {
 
@@ -40,7 +39,7 @@ public void testNonBooleanFeatureFlag() {
     }
 
     public void testBooleanFeatureFlagWithDefaultSetToFalse() {
-        final String testFlag = IDENTITY;
+        final String testFlag = EXTENSIONS;
         FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
         assertNotNull(testFlag);
         assertFalse(FeatureFlags.isEnabled(testFlag));
@@ -49,15 +48,13 @@ public void testBooleanFeatureFlagWithDefaultSetToFalse() {
     public void testBooleanFeatureFlagInitializedWithEmptySettingsAndDefaultSetToFalse() {
         final String testFlag = DATETIME_FORMATTER_CACHING;
         FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
-        assertNotNull(testFlag);
         assertFalse(FeatureFlags.isEnabled(testFlag));
     }
 
     public void testInitializeFeatureFlagsWithExperimentalSettings() {
-        FeatureFlags.initializeFeatureFlags(Settings.builder().put(IDENTITY, true).build());
-        assertTrue(FeatureFlags.isEnabled(IDENTITY));
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(EXTENSIONS, true).build());
+        assertTrue(FeatureFlags.isEnabled(EXTENSIONS));
         assertFalse(FeatureFlags.isEnabled(DATETIME_FORMATTER_CACHING));
-        assertFalse(FeatureFlags.isEnabled(EXTENSIONS));
         // reset FeatureFlags to defaults
         FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
     }
diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java
index 5ae1bdce48cd5..bf1d52b49cb1f 100644
--- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java
+++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java
@@ -153,8 +153,7 @@ public List<Setting<?>> getExtensionSettings() {
             null,
             new NodeClient(Settings.EMPTY, threadPool),
             new NoneCircuitBreakerService(),
-            new UsageService(),
-            new IdentityService(Settings.EMPTY, threadPool, List.of())
+            new UsageService()
         );
         when(actionModule.getDynamicActionRegistry()).thenReturn(mock(DynamicActionRegistry.class));
         when(actionModule.getRestController()).thenReturn(restController);
diff --git a/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java b/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java
index db97c3ece94ba..c25150873a1ce 100644
--- a/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java
+++ b/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java
@@ -35,10 +35,13 @@
 import org.opensearch.Version;
 import org.opensearch.action.FailedNodeException;
 import org.opensearch.action.support.nodes.BaseNodeResponse;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.telemetry.metrics.Counter;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -54,6 +57,12 @@
 import static java.util.Collections.emptyMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.anyDouble;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class AsyncShardFetchTests extends OpenSearchTestCase {
     private final DiscoveryNode node1 = new DiscoveryNode(
@@ -78,13 +87,29 @@ public class AsyncShardFetchTests extends OpenSearchTestCase {
 
     private ThreadPool threadPool;
     private TestFetch test;
+    private Counter asyncFetchSuccessCounter;
+    private Counter asyncFetchFailureCounter;
+    private Counter dummyCounter;
 
     @Override
     @Before
     public void setUp() throws Exception {
         super.setUp();
         this.threadPool = new TestThreadPool(getTestName());
-        this.test = new TestFetch(threadPool);
+        final MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
+        this.asyncFetchFailureCounter = mock(Counter.class);
+        this.asyncFetchSuccessCounter = mock(Counter.class);
+        this.dummyCounter = mock(Counter.class);
+        when(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer(invocationOnMock -> {
+            String counterName = (String) invocationOnMock.getArguments()[0];
+            if (counterName.contains("async.fetch.success.count")) {
+                return asyncFetchSuccessCounter;
+            } else if (counterName.contains("async.fetch.failure.count")) {
+                return asyncFetchFailureCounter;
+            }
+            return dummyCounter;
+        });
+        this.test = new TestFetch(threadPool, metricsRegistry);
     }
 
     @After
@@ -100,14 +125,26 @@ public void testClose() throws Exception {
         AsyncShardFetch.FetchResult<Response> fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
         assertThat(test.reroute.get(), equalTo(0));
+        // counter remains 0 because fetch is ongoing
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // fire a response, wait on reroute incrementing
         test.fireSimulationAndWait(node1.getId());
+        // counter goes up because fetch completed
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
+
         // verify we get back the data node
         assertThat(test.reroute.get(), equalTo(1));
         test.close();
         try {
             test.fetchData(nodes, emptyMap());
+            // counter should not go up when calling fetchData since fetch never completed
+            verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+            verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+            verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
             fail("fetch data should fail when closed");
         } catch (IllegalStateException e) {
             // all is well
@@ -125,12 +162,21 @@ public void testFullCircleSingleNodeSuccess() throws Exception {
 
         // fire a response, wait on reroute incrementing
         test.fireSimulationAndWait(node1.getId());
+        // total counter goes up by 1 after success
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
+
         // verify we get back the data node
         assertThat(test.reroute.get(), equalTo(1));
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(1));
         assertThat(fetchData.getData().get(node1), sameInstance(response1));
+        // counter remains same because fetchData does not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
     }
 
     public void testFullCircleSingleNodeFailure() throws Exception {
@@ -145,24 +191,47 @@ public void testFullCircleSingleNodeFailure() throws Exception {
 
         // fire a response, wait on reroute incrementing
         test.fireSimulationAndWait(node1.getId());
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(1.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
+
         // failure, fetched data exists, but has no data
         assertThat(test.reroute.get(), equalTo(1));
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(0));
+        // counter remains same because fetchData does not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(1.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
 
         // on failure, we reset the failure on a successive call to fetchData, and try again afterwards
         test.addSimulation(node1.getId(), response1);
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
+        // No additional failure, empty data so no change in counter
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(1.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
 
         test.fireSimulationAndWait(node1.getId());
+        // Success counter will increase
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(1.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
+
         // 2 reroutes, cause we have a failure that we clear
         assertThat(test.reroute.get(), equalTo(3));
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(1));
         assertThat(fetchData.getData().get(node1), sameInstance(response1));
+        // counter remains same because fetchData does not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(1.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
     }
 
     public void testIgnoreResponseFromDifferentRound() throws Exception {
@@ -173,20 +242,40 @@ public void testIgnoreResponseFromDifferentRound() throws Exception {
         AsyncShardFetch.FetchResult<Response> fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
         assertThat(test.reroute.get(), equalTo(0));
+        // counter 0 because fetchData is not completed
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // handle a response with incorrect round id, wait on reroute incrementing
         test.processAsyncFetch(Collections.singletonList(response1), Collections.emptyList(), 0);
         assertThat(fetchData.hasData(), equalTo(false));
         assertThat(test.reroute.get(), equalTo(1));
+        // success counter increments to 1 because we called processAsyncFetch with a valid response, even though the round was incorrect
+        // failure counter also increments by 1 with empty list
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(0.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
 
         // fire a response (with correct round id), wait on reroute incrementing
         test.fireSimulationAndWait(node1.getId());
+        // success counter now goes up by 1 because fetchData completed
+        verify(asyncFetchSuccessCounter, times(2)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(2)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(0.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
+
         // verify we get back the data node
         assertThat(test.reroute.get(), equalTo(2));
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(1));
         assertThat(fetchData.getData().get(node1), sameInstance(response1));
+        // total counter remains same because fetchdata does not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(2)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(2)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(0.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
     }
 
     public void testIgnoreFailureFromDifferentRound() throws Exception {
@@ -198,6 +287,9 @@ public void testIgnoreFailureFromDifferentRound() throws Exception {
         AsyncShardFetch.FetchResult<Response> fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
         assertThat(test.reroute.get(), equalTo(0));
+        // counter 0 because fetchData still ongoing
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // handle a failure with incorrect round id, wait on reroute incrementing
         test.processAsyncFetch(
@@ -207,14 +299,30 @@ public void testIgnoreFailureFromDifferentRound() throws Exception {
         );
         assertThat(fetchData.hasData(), equalTo(false));
         assertThat(test.reroute.get(), equalTo(1));
+        // success counter called with empty list
+        // failure counter goes up by 1 because of the failure
+        verify(asyncFetchSuccessCounter, times(1)).add(0.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(1.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
 
         // fire a response, wait on reroute incrementing
         test.fireSimulationAndWait(node1.getId());
+        // failure counter goes up by 1 because of the failure
+        verify(asyncFetchSuccessCounter, times(1)).add(0.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(2)).add(1.0);
+        verify(asyncFetchFailureCounter, times(2)).add(anyDouble());
         // failure, fetched data exists, but has no data
         assertThat(test.reroute.get(), equalTo(2));
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(0));
+        // counters remain same because fetchData does not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(1)).add(0.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(2)).add(1.0);
+        verify(asyncFetchFailureCounter, times(2)).add(anyDouble());
     }
 
     public void testTwoNodesOnSetup() throws Exception {
@@ -226,16 +334,32 @@ public void testTwoNodesOnSetup() throws Exception {
         AsyncShardFetch.FetchResult<Response> fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
         assertThat(test.reroute.get(), equalTo(0));
+        // counter 0 because fetch ongoing
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // fire the first response, it should trigger a reroute
         test.fireSimulationAndWait(node1.getId());
+        // counter 1 because one fetch completed
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
+
         // there is still another on going request, so no data
         assertThat(test.getNumberOfInFlightFetches(), equalTo(1));
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
+        // counter still 1 because fetchData did not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // fire the second simulation, this should allow us to get the data
         test.fireSimulationAndWait(node2.getId());
+        // counter 2 because 2 fetches completed
+        verify(asyncFetchSuccessCounter, times(2)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(2)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
         // no more ongoing requests, we should fetch the data
         assertThat(test.reroute.get(), equalTo(2));
         fetchData = test.fetchData(nodes, emptyMap());
@@ -243,6 +367,10 @@ public void testTwoNodesOnSetup() throws Exception {
         assertThat(fetchData.getData().size(), equalTo(2));
         assertThat(fetchData.getData().get(node1), sameInstance(response1));
         assertThat(fetchData.getData().get(node2), sameInstance(response2));
+        // counter still 2 because fetchData call did not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(2)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(2)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
     }
 
     public void testTwoNodesOnSetupAndFailure() throws Exception {
@@ -254,34 +382,59 @@ public void testTwoNodesOnSetupAndFailure() throws Exception {
         AsyncShardFetch.FetchResult<Response> fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
         assertThat(test.reroute.get(), equalTo(0));
+        // counter 0 because both fetches ongoing
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // fire the first response, it should trigger a reroute
         test.fireSimulationAndWait(node1.getId());
         assertThat(test.reroute.get(), equalTo(1));
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
+        // counter 1 because one fetch completed
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // fire the second simulation, this should allow us to get the data
         test.fireSimulationAndWait(node2.getId());
+        // failure counter up by 1 because one fetch failed
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(1.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
         assertThat(test.reroute.get(), equalTo(2));
+
         // since one of those failed, we should only have one entry
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(1));
         assertThat(fetchData.getData().get(node1), sameInstance(response1));
+        // success and failure counters same because fetchData did not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(1)).add(1.0);
+        verify(asyncFetchFailureCounter, times(1)).add(anyDouble());
     }
 
     public void testTwoNodesAddedInBetween() throws Exception {
         DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).build();
         test.addSimulation(node1.getId(), response1);
 
-        // no fetched data, 2 requests still on going
+        // no fetched data, request still on going
         AsyncShardFetch.FetchResult<Response> fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
         assertThat(test.reroute.get(), equalTo(0));
+        // counter 0 because both fetches ongoing
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // fire the first response, it should trigger a reroute
         test.fireSimulationAndWait(node1.getId());
+        // counter 1 because fetch completed
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // now, add a second node to the nodes, it should add it to the ongoing requests
         nodes = DiscoveryNodes.builder(nodes).add(node2).build();
@@ -289,16 +442,28 @@ public void testTwoNodesAddedInBetween() throws Exception {
         // no fetch data, has a new node introduced
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
+        // counter still 1 because second fetch ongoing
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // fire the second simulation, this should allow us to get the data
         test.fireSimulationAndWait(node2.getId());
+        // counter now 2 because 2 fetches completed
+        verify(asyncFetchSuccessCounter, times(2)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(2)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
-        // since one of those failed, we should only have one entry
+        // since both succeeded, we should have 2 entries
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(2));
         assertThat(fetchData.getData().get(node1), sameInstance(response1));
         assertThat(fetchData.getData().get(node2), sameInstance(response2));
+        // counter still 2 because fetchData did not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(2)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(2)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
     }
 
     public void testClearCache() throws Exception {
@@ -312,21 +477,36 @@ public void testClearCache() throws Exception {
         AsyncShardFetch.FetchResult<Response> fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
         assertThat(test.reroute.get(), equalTo(0));
+        // counter 0 because fetch ongoing
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         test.fireSimulationAndWait(node1.getId());
         assertThat(test.reroute.get(), equalTo(1));
+        // counter 1 because 1 fetch completed
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // verify we get back right data from node
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(1));
         assertThat(fetchData.getData().get(node1), sameInstance(response1));
+        // counter still 1 because a new fetch is not called
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // second fetch gets same data
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(1));
         assertThat(fetchData.getData().get(node1), sameInstance(response1));
+        // counter still 1 because a new fetch is not called
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         test.clearCacheForNode(node1.getId());
 
@@ -336,15 +516,27 @@ public void testClearCache() throws Exception {
         // no fetched data, new request on going
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
+        // counter still 1 because new fetch is still ongoing
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         test.fireSimulationAndWait(node1.getId());
         assertThat(test.reroute.get(), equalTo(2));
+        // counter now 2 because second fetch completed
+        verify(asyncFetchSuccessCounter, times(2)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(2)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // verify we get new data back
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(1));
         assertThat(fetchData.getData().get(node1), sameInstance(response1_2));
+        // counter still 2 because fetchData did not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(2)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(2)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
     }
 
     public void testConcurrentRequestAndClearCache() throws Exception {
@@ -355,12 +547,19 @@ public void testConcurrentRequestAndClearCache() throws Exception {
         AsyncShardFetch.FetchResult<Response> fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
         assertThat(test.reroute.get(), equalTo(0));
+        // counter 0 because fetch ongoing
+        verify(asyncFetchSuccessCounter, times(0)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // clear cache while request is still on going, before it is processed
         test.clearCacheForNode(node1.getId());
 
         test.fireSimulationAndWait(node1.getId());
         assertThat(test.reroute.get(), equalTo(1));
+        // counter 1 because fetch completed, even though cache was wiped
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // prepare next request
         test.addSimulation(node1.getId(), response1_2);
@@ -368,15 +567,27 @@ public void testConcurrentRequestAndClearCache() throws Exception {
         // verify still no fetched data, request still on going
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(false));
+        // counter unchanged because fetch ongoing
+        verify(asyncFetchSuccessCounter, times(1)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(1)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         test.fireSimulationAndWait(node1.getId());
         assertThat(test.reroute.get(), equalTo(2));
+        // counter 2 because second fetch completed
+        verify(asyncFetchSuccessCounter, times(2)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(2)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
         // verify we get new data back
         fetchData = test.fetchData(nodes, emptyMap());
         assertThat(fetchData.hasData(), equalTo(true));
         assertThat(fetchData.getData().size(), equalTo(1));
         assertThat(fetchData.getData().get(node1), sameInstance(response1_2));
+        // counter unchanged because fetchData does not trigger new async fetch
+        verify(asyncFetchSuccessCounter, times(2)).add(1.0);
+        verify(asyncFetchSuccessCounter, times(2)).add(anyDouble());
+        verify(asyncFetchFailureCounter, times(0)).add(anyDouble());
 
     }
 
@@ -398,8 +609,15 @@ static class Entry {
         private final Map<String, Entry> simulations = new ConcurrentHashMap<>();
         private AtomicInteger reroute = new AtomicInteger();
 
-        TestFetch(ThreadPool threadPool) {
-            super(LogManager.getLogger(TestFetch.class), "test", new ShardId("test", "_na_", 1), "", null);
+        TestFetch(ThreadPool threadPool, MetricsRegistry metricsRegistry) {
+            super(
+                LogManager.getLogger(TestFetch.class),
+                "test",
+                new ShardId("test", "_na_", 1),
+                "",
+                null,
+                new ClusterManagerMetrics(metricsRegistry)
+            );
             this.threadPool = threadPool;
         }
 
diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
index 5ea5241762753..efdb3076f419c 100644
--- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
+++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
@@ -1244,14 +1244,72 @@ public void testGatewayForRemoteStateForInitialBootstrapBlocksApplied() throws I
         }
     }
 
-    private MockGatewayMetaState newGatewayForRemoteState(
+    public void testGatewayMetaStateRemoteStateDownloadRetries() throws IOException {
+        MockGatewayMetaState gateway = null;
+        MockGatewayMetaState gatewayMetaStateSpy = null;
+        try {
+            RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class);
+            when(remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster")).thenReturn("test-cluster-uuid");
+            RemoteStoreRestoreService remoteStoreRestoreService = mock(RemoteStoreRestoreService.class);
+            when(remoteStoreRestoreService.restore(any(), any(), anyBoolean(), any())).thenThrow(
+                new IllegalStateException("unable to download cluster state")
+            ).thenReturn(RemoteRestoreResult.build("test-cluster-uuid", null, ClusterState.EMPTY_STATE));
+            final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry();
+            gateway = initializeGatewayForRemoteState(true);
+            gatewayMetaStateSpy = Mockito.spy(gateway);
+            startGatewayForRemoteState(
+                gatewayMetaStateSpy,
+                remoteClusterStateService,
+                remoteStoreRestoreService,
+                persistedStateRegistry,
+                ClusterState.EMPTY_STATE
+            );
+            verify(gatewayMetaStateSpy, times(2)).restoreClusterState(Mockito.any(), Mockito.any(), Mockito.any());
+        } finally {
+            IOUtils.close(gatewayMetaStateSpy);
+        }
+    }
+
+    public void testGatewayMetaStateRemoteStateDownloadFailure() throws IOException {
+        MockGatewayMetaState gateway = null;
+        final MockGatewayMetaState gatewayMetaStateSpy;
+        try {
+            RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class);
+            when(remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster")).thenReturn("test-cluster-uuid");
+            RemoteStoreRestoreService remoteStoreRestoreService = mock(RemoteStoreRestoreService.class);
+            when(remoteStoreRestoreService.restore(any(), any(), anyBoolean(), any())).thenThrow(
+                new IllegalStateException("unable to download cluster state")
+            );
+            final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry();
+            gateway = initializeGatewayForRemoteState(true);
+            gatewayMetaStateSpy = Mockito.spy(gateway);
+            assertThrows(
+                Error.class,
+                () -> startGatewayForRemoteState(
+                    gatewayMetaStateSpy,
+                    remoteClusterStateService,
+                    remoteStoreRestoreService,
+                    persistedStateRegistry,
+                    ClusterState.EMPTY_STATE
+                )
+            );
+            verify(gatewayMetaStateSpy, times(5)).restoreClusterState(Mockito.any(), Mockito.any(), Mockito.any());
+        } finally {
+            IOUtils.close(gateway);
+        }
+    }
+
+    private MockGatewayMetaState initializeGatewayForRemoteState(boolean prepareFullState) {
+        return new MockGatewayMetaState(localNode, bigArrays, prepareFullState);
+    }
+
+    private MockGatewayMetaState startGatewayForRemoteState(
+        MockGatewayMetaState gateway,
         RemoteClusterStateService remoteClusterStateService,
         RemoteStoreRestoreService remoteStoreRestoreService,
         PersistedStateRegistry persistedStateRegistry,
-        ClusterState currentState,
-        boolean prepareFullState
+        ClusterState currentState
     ) throws IOException {
-        MockGatewayMetaState gateway = new MockGatewayMetaState(localNode, bigArrays, prepareFullState);
         String randomRepoName = "randomRepoName";
         String stateRepoTypeAttributeKey = String.format(
             Locale.getDefault(),
@@ -1305,6 +1363,24 @@ private MockGatewayMetaState newGatewayForRemoteState(
         return gateway;
     }
 
+    private MockGatewayMetaState newGatewayForRemoteState(
+        RemoteClusterStateService remoteClusterStateService,
+        RemoteStoreRestoreService remoteStoreRestoreService,
+        PersistedStateRegistry persistedStateRegistry,
+        ClusterState currentState,
+        boolean prepareFullState
+    ) throws IOException {
+        MockGatewayMetaState gatewayMetaState = initializeGatewayForRemoteState(prepareFullState);
+        startGatewayForRemoteState(
+            gatewayMetaState,
+            remoteClusterStateService,
+            remoteStoreRestoreService,
+            persistedStateRegistry,
+            currentState
+        );
+        return gatewayMetaState;
+    }
+
     private static BigArrays getBigArrays() {
         return usually()
             ? BigArrays.NON_RECYCLING_INSTANCE
diff --git a/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java b/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java
index 12030ad41d508..39c4ee8c8ca06 100644
--- a/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java
+++ b/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.gateway;
 
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.OpenSearchAllocationTestCase;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
@@ -19,6 +20,7 @@
 import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.GatewayStartedShard;
 import org.opensearch.gateway.TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch;
 import org.opensearch.indices.store.ShardAttributes;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -52,7 +54,8 @@ public void setupShardBatchCache(String batchId, int numberOfShards) {
             GatewayStartedShard.class,
             new GatewayStartedShard(null, false, null, null),
             GatewayStartedShard::isEmpty,
-            new ShardBatchResponseFactory<>(true)
+            new ShardBatchResponseFactory<>(true),
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
         );
     }
 
diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
index 3f9aa1245cab3..09c2933680be3 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
@@ -34,6 +34,9 @@
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute;
 import org.opensearch.test.EqualsHashCodeTestUtils;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -64,6 +67,14 @@
 
 public class ClusterMetadataManifestTests extends OpenSearchTestCase {
 
+    private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+
+    @After
+    public void teardown() throws Exception {
+        super.tearDown();
+        threadPool.shutdown();
+    }
+
     public void testClusterMetadataManifestXContentV0() throws IOException {
         UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V0);
         ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
@@ -214,7 +225,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() {
                     "indicesRoutingDiffPath"
                 )
             )
-            .checksum(new ClusterStateChecksum(createClusterState()))
+            .checksum(new ClusterStateChecksum(createClusterState(), threadPool))
             .build();
         {  // Mutate Cluster Term
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(
@@ -647,7 +658,7 @@ public void testClusterMetadataManifestXContentV4() throws IOException {
         UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
         UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute");
         final StringKeyDiffProvider<IndexRoutingTable> routingTableIncrementalDiff = Mockito.mock(StringKeyDiffProvider.class);
-        ClusterStateChecksum checksum = new ClusterStateChecksum(createClusterState());
+        ClusterStateChecksum checksum = new ClusterStateChecksum(createClusterState(), threadPool);
         ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
             .clusterTerm(1L)
             .stateVersion(1L)
diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java
index 0203e56dd2d5c..9b98187053a39 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java
@@ -34,6 +34,9 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
 
 import java.io.IOException;
 import java.util.EnumSet;
@@ -41,14 +44,21 @@
 import java.util.Map;
 
 public class ClusterStateChecksumTests extends OpenSearchTestCase {
+    private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+
+    @After
+    public void teardown() throws Exception {
+        super.tearDown();
+        threadPool.shutdown();
+    }
 
     public void testClusterStateChecksumEmptyClusterState() {
-        ClusterStateChecksum checksum = new ClusterStateChecksum(ClusterState.EMPTY_STATE);
+        ClusterStateChecksum checksum = new ClusterStateChecksum(ClusterState.EMPTY_STATE, threadPool);
         assertNotNull(checksum);
     }
 
     public void testClusterStateChecksum() {
-        ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState());
+        ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool);
         assertNotNull(checksum);
         assertTrue(checksum.routingTableChecksum != 0);
         assertTrue(checksum.nodesChecksum != 0);
@@ -65,8 +75,8 @@ public void testClusterStateChecksum() {
     }
 
     public void testClusterStateMatchChecksum() {
-        ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState());
-        ClusterStateChecksum newChecksum = new ClusterStateChecksum(generateClusterState());
+        ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool);
+        ClusterStateChecksum newChecksum = new ClusterStateChecksum(generateClusterState(), threadPool);
         assertNotNull(checksum);
         assertNotNull(newChecksum);
         assertEquals(checksum.routingTableChecksum, newChecksum.routingTableChecksum);
@@ -84,7 +94,7 @@ public void testClusterStateMatchChecksum() {
     }
 
     public void testXContentConversion() throws IOException {
-        ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState());
+        ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool);
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
         checksum.toXContent(builder, ToXContent.EMPTY_PARAMS);
@@ -97,7 +107,7 @@ public void testXContentConversion() throws IOException {
     }
 
     public void testSerialization() throws IOException {
-        ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState());
+        ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool);
         BytesStreamOutput output = new BytesStreamOutput();
         checksum.writeTo(output);
 
@@ -109,10 +119,10 @@ public void testSerialization() throws IOException {
 
     public void testGetMismatchEntities() {
         ClusterState clsState1 = generateClusterState();
-        ClusterStateChecksum checksum = new ClusterStateChecksum(clsState1);
+        ClusterStateChecksum checksum = new ClusterStateChecksum(clsState1, threadPool);
         assertTrue(checksum.getMismatchEntities(checksum).isEmpty());
 
-        ClusterStateChecksum checksum2 = new ClusterStateChecksum(clsState1);
+        ClusterStateChecksum checksum2 = new ClusterStateChecksum(clsState1, threadPool);
         assertTrue(checksum.getMismatchEntities(checksum2).isEmpty());
 
         ClusterState clsState2 = ClusterState.builder(ClusterName.DEFAULT)
@@ -122,7 +132,7 @@ public void testGetMismatchEntities() {
             .customs(Map.of())
             .metadata(Metadata.EMPTY_METADATA)
             .build();
-        ClusterStateChecksum checksum3 = new ClusterStateChecksum(clsState2);
+        ClusterStateChecksum checksum3 = new ClusterStateChecksum(clsState2, threadPool);
         List<String> mismatches = checksum.getMismatchEntities(checksum3);
         assertFalse(mismatches.isEmpty());
         assertEquals(11, mismatches.size());
@@ -151,8 +161,8 @@ public void testGetMismatchEntitiesUnorderedInput() {
         ClusterState state2 = ClusterState.builder(state1).nodes(nodes1).build();
         ClusterState state3 = ClusterState.builder(state1).nodes(nodes2).build();
 
-        ClusterStateChecksum checksum1 = new ClusterStateChecksum(state2);
-        ClusterStateChecksum checksum2 = new ClusterStateChecksum(state3);
+        ClusterStateChecksum checksum1 = new ClusterStateChecksum(state2, threadPool);
+        ClusterStateChecksum checksum2 = new ClusterStateChecksum(state3, threadPool);
         assertEquals(checksum2, checksum1);
     }
 
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index b11d5e48bec06..35a8ae16cacf7 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -3123,7 +3123,7 @@ public void testWriteFullMetadataSuccessWithChecksumValidationEnabled() throws I
             .previousClusterUUID("prev-cluster-uuid")
             .routingTableVersion(1L)
             .indicesRouting(List.of(uploadedIndiceRoutingMetadata))
-            .checksum(new ClusterStateChecksum(clusterState))
+            .checksum(new ClusterStateChecksum(clusterState, threadPool))
             .build();
 
         assertThat(manifest.getIndices().size(), is(1));
@@ -3193,7 +3193,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationEnabled() t
 
         final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
             .indices(Collections.emptyList())
-            .checksum(new ClusterStateChecksum(clusterState))
+            .checksum(new ClusterStateChecksum(clusterState, threadPool))
             .build();
         when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path"));
 
@@ -3219,7 +3219,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationEnabled() t
             .previousClusterUUID("prev-cluster-uuid")
             .routingTableVersion(1)
             .indicesRouting(List.of(uploadedIndiceRoutingMetadata))
-            .checksum(new ClusterStateChecksum(clusterState))
+            .checksum(new ClusterStateChecksum(clusterState, threadPool))
             .build();
 
         assertThat(manifest.getIndices().size(), is(1));
@@ -3245,7 +3245,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationModeNone()
 
         final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
             .indices(Collections.emptyList())
-            .checksum(new ClusterStateChecksum(clusterState))
+            .checksum(new ClusterStateChecksum(clusterState, threadPool))
             .build();
         when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path"));
 
@@ -3271,7 +3271,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationModeNone()
             .previousClusterUUID("prev-cluster-uuid")
             .routingTableVersion(1)
             .indicesRouting(List.of(uploadedIndiceRoutingMetadata))
-            .checksum(new ClusterStateChecksum(clusterState))
+            .checksum(new ClusterStateChecksum(clusterState, threadPool))
             .build();
 
         assertThat(manifest.getIndices().size(), is(1));
@@ -3342,13 +3342,14 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullC
             anyString(),
             anyBoolean()
         );
+        assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed());
     }
 
     public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws IOException {
         initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE);
         ClusterState clusterState = generateClusterStateWithAllAttributes().build();
         ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum(
-            new ClusterStateChecksum(clusterState)
+            new ClusterStateChecksum(clusterState, threadPool)
         ).build();
         remoteClusterStateService.start();
         RemoteClusterStateService mockService = spy(remoteClusterStateService);
@@ -3374,13 +3375,14 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws
             );
         mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true);
         verify(mockService, times(1)).validateClusterStateFromChecksum(manifest, clusterState, ClusterName.DEFAULT.value(), NODE_ID, true);
+        assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed());
     }
 
     public void testGetClusterStateForManifestWithChecksumValidationModeNone() throws IOException {
         initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.NONE);
         ClusterState clusterState = generateClusterStateWithAllAttributes().build();
         ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum(
-            new ClusterStateChecksum(clusterState)
+            new ClusterStateChecksum(clusterState, threadPool)
         ).build();
         remoteClusterStateService.start();
         RemoteClusterStateService mockService = spy(remoteClusterStateService);
@@ -3406,13 +3408,14 @@ public void testGetClusterStateForManifestWithChecksumValidationModeNone() throw
             );
         mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true);
         verify(mockService, times(0)).validateClusterStateFromChecksum(any(), any(), anyString(), anyString(), anyBoolean());
+        assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed());
     }
 
     public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMismatch() throws IOException {
         initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE);
         ClusterState clusterState = generateClusterStateWithAllAttributes().build();
         ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum(
-            new ClusterStateChecksum(clusterState)
+            new ClusterStateChecksum(clusterState, threadPool)
         ).build();
         remoteClusterStateService.start();
         RemoteClusterStateService mockService = spy(remoteClusterStateService);
@@ -3448,6 +3451,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma
             NODE_ID,
             true
         );
+        assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed());
     }
 
     public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatch() throws IOException {
@@ -3461,7 +3465,7 @@ public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatc
         );
         ClusterState clusterState = generateClusterStateWithAllAttributes().build();
         ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum(
-            new ClusterStateChecksum(clusterState)
+            new ClusterStateChecksum(clusterState, threadPool)
         ).build();
         remoteClusterStateService.start();
         RemoteClusterStateService mockService = spy(remoteClusterStateService);
@@ -3494,13 +3498,14 @@ public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatc
             NODE_ID,
             true
         );
+        assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed());
     }
 
     public void testGetClusterStateUsingDiffWithChecksum() throws IOException {
         initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE);
         ClusterState clusterState = generateClusterStateWithAllAttributes().build();
         ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum(
-            new ClusterStateChecksum(clusterState)
+            new ClusterStateChecksum(clusterState, threadPool)
         ).diffManifest(ClusterStateDiffManifest.builder().build()).build();
 
         remoteClusterStateService.start();
@@ -3535,13 +3540,14 @@ public void testGetClusterStateUsingDiffWithChecksum() throws IOException {
             eq(NODE_ID),
             eq(false)
         );
+        assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed());
     }
 
     public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOException {
         initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.NONE);
         ClusterState clusterState = generateClusterStateWithAllAttributes().build();
         ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum(
-            new ClusterStateChecksum(clusterState)
+            new ClusterStateChecksum(clusterState, threadPool)
         ).diffManifest(ClusterStateDiffManifest.builder().build()).build();
 
         remoteClusterStateService.start();
@@ -3576,13 +3582,14 @@ public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOExceptio
             eq(NODE_ID),
             eq(false)
         );
+        assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed());
     }
 
     public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws IOException {
         initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.DEBUG);
         ClusterState clusterState = generateClusterStateWithAllAttributes().build();
         ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum(
-            new ClusterStateChecksum(clusterState)
+            new ClusterStateChecksum(clusterState, threadPool)
         ).diffManifest(ClusterStateDiffManifest.builder().build()).build();
 
         remoteClusterStateService.start();
@@ -3616,13 +3623,14 @@ public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws I
             eq(NODE_ID),
             eq(false)
         );
+        assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed());
     }
 
     public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws IOException {
         initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.TRACE);
         ClusterState clusterState = generateClusterStateWithAllAttributes().build();
         ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum(
-            new ClusterStateChecksum(clusterState)
+            new ClusterStateChecksum(clusterState, threadPool)
         ).diffManifest(ClusterStateDiffManifest.builder().build()).build();
 
         remoteClusterStateService.start();
@@ -3677,13 +3685,14 @@ public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws I
             eq(NODE_ID),
             eq(false)
         );
+        assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed());
     }
 
     public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOException {
         initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE);
         ClusterState clusterState = generateClusterStateWithAllAttributes().build();
         ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum(
-            new ClusterStateChecksum(clusterState)
+            new ClusterStateChecksum(clusterState, threadPool)
         ).diffManifest(ClusterStateDiffManifest.builder().build()).build();
 
         remoteClusterStateService.start();
@@ -3738,6 +3747,7 @@ public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOExceptio
             eq(NODE_ID),
             eq(false)
         );
+        assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed());
     }
 
     private void mockObjectsForGettingPreviousClusterUUID(Map<String, String> clusterUUIDsPointers) throws IOException {
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java
index b7395b993f67b..7cae1cd25ee93 100644
--- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java
@@ -54,7 +54,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments(
 
             for (int i = 0; i < dimensionsSplitOrder.size(); i++) {
                 String dimension = dimensionsSplitOrder.get(i).getField();
-                dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension));
+                dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension));
             }
 
             List<SequentialDocValuesIterator> metricReaders = new ArrayList<>();
@@ -69,7 +69,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments(
                         metric.getField(),
                         metricStat.getTypeName()
                     );
-                    metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName)));
+                    metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName)));
 
                 }
             }
@@ -92,7 +92,7 @@ public static StarTreeDocument getStarTreeDocument(
         Long[] dims = new Long[dimensionReaders.length];
         int i = 0;
         for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) {
-            dimensionDocValueIterator.nextDoc(currentDocId);
+            dimensionDocValueIterator.nextEntry(currentDocId);
             Long val = dimensionDocValueIterator.value(currentDocId);
             dims[i] = val;
             i++;
@@ -100,7 +100,7 @@ public static StarTreeDocument getStarTreeDocument(
         i = 0;
         Object[] metrics = new Object[metricReaders.size()];
         for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) {
-            metricDocValuesIterator.nextDoc(currentDocId);
+            metricDocValuesIterator.nextEntry(currentDocId);
             metrics[i] = toAggregatorValueType(metricDocValuesIterator.value(currentDocId), fieldValueConverters.get(i));
             i++;
         }
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java
index 65adc43ea8bea..b77200f173e71 100644
--- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java
@@ -53,6 +53,8 @@
 import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType;
 import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator;
 import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils;
+import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator;
+import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator;
 import org.opensearch.index.mapper.ContentPath;
 import org.opensearch.index.mapper.DocumentMapper;
 import org.opensearch.index.mapper.FieldValueConverter;
@@ -395,7 +397,9 @@ SequentialDocValuesIterator[] getDimensionIterators(StarTreeDocument[] starTreeD
                     docsWithField.add(i);
                 }
             }
-            sequentialDocValuesIterators[j] = new SequentialDocValuesIterator(getSortedNumericMock(dimList, docsWithField));
+            sequentialDocValuesIterators[j] = new SequentialDocValuesIterator(
+                new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField))
+            );
         }
         return sequentialDocValuesIterators;
     }
@@ -412,7 +416,9 @@ List<SequentialDocValuesIterator> getMetricIterators(StarTreeDocument[] starTree
                     docsWithField.add(i);
                 }
             }
-            sequentialDocValuesIterators.add(new SequentialDocValuesIterator(getSortedNumericMock(metricslist, docsWithField)));
+            sequentialDocValuesIterators.add(
+                new SequentialDocValuesIterator(new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricslist, docsWithField)))
+            );
         }
         return sequentialDocValuesIterators;
     }
@@ -1985,10 +1991,16 @@ public void testFlushFlow() throws IOException {
         List<Integer> metricsWithField = List.of(0, 1, 2, 3, 4, 5);
 
         compositeField = getStarTreeFieldWithMultipleMetrics();
-        SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField);
-        SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2);
-        SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField);
-        SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField);
+        SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField));
+        SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator(
+            getSortedNumericMock(dimList2, docsWithField2)
+        );
+        SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator(
+            getSortedNumericMock(metricsList, metricsWithField)
+        );
+        SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator(
+            getSortedNumericMock(metricsList, metricsWithField)
+        );
 
         writeState = getWriteState(6, writeState.segmentInfo.getId());
         builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService);
@@ -2081,10 +2093,16 @@ public void testFlushFlowDimsReverse() throws IOException {
         List<Integer> metricsWithField = List.of(0, 1, 2, 3, 4, 5);
 
         compositeField = getStarTreeFieldWithMultipleMetrics();
-        SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField);
-        SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2);
-        SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField);
-        SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField);
+        SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField));
+        SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator(
+            getSortedNumericMock(dimList2, docsWithField2)
+        );
+        SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator(
+            getSortedNumericMock(metricsList, metricsWithField)
+        );
+        SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator(
+            getSortedNumericMock(metricsList, metricsWithField)
+        );
 
         writeState = getWriteState(6, writeState.segmentInfo.getId());
         this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec(
@@ -2508,9 +2526,14 @@ private StarTreeValues getStarTreeValues(
         SortedNumericDocValues d1sndv = dimList;
         SortedNumericDocValues d2sndv = dimList2;
         SortedNumericDocValues m1sndv = metricsList;
-        Map<String, Supplier<DocIdSetIterator>> dimDocIdSetIterators = Map.of("field1", () -> d1sndv, "field3", () -> d2sndv);
+        Map<String, Supplier<StarTreeValuesIterator>> dimDocIdSetIterators = Map.of(
+            "field1",
+            () -> new SortedNumericStarTreeValuesIterator(d1sndv),
+            "field3",
+            () -> new SortedNumericStarTreeValuesIterator(d2sndv)
+        );
 
-        Map<String, Supplier<DocIdSetIterator>> metricDocIdSetIterators = new LinkedHashMap<>();
+        Map<String, Supplier<StarTreeValuesIterator>> metricDocIdSetIterators = new LinkedHashMap<>();
         for (Metric metric : sf.getMetrics()) {
             for (MetricStat metricStat : metric.getMetrics()) {
                 String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues(
@@ -2518,7 +2541,7 @@ private StarTreeValues getStarTreeValues(
                     metric.getField(),
                     metricStat.getTypeName()
                 );
-                metricDocIdSetIterators.put(metricFullName, () -> m1sndv);
+                metricDocIdSetIterators.put(metricFullName, () -> new SortedNumericStarTreeValuesIterator(m1sndv));
             }
         }
 
@@ -3648,18 +3671,18 @@ private StarTreeValues getStarTreeValues(
         SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4);
         SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField);
         SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList1, metricsWithField1);
-        Map<String, Supplier<DocIdSetIterator>> dimDocIdSetIterators = Map.of(
+        Map<String, Supplier<StarTreeValuesIterator>> dimDocIdSetIterators = Map.of(
             "field1",
-            () -> d1sndv,
+            () -> new SortedNumericStarTreeValuesIterator(d1sndv),
             "field3",
-            () -> d2sndv,
+            () -> new SortedNumericStarTreeValuesIterator(d2sndv),
             "field5",
-            () -> d3sndv,
+            () -> new SortedNumericStarTreeValuesIterator(d3sndv),
             "field8",
-            () -> d4sndv
+            () -> new SortedNumericStarTreeValuesIterator(d4sndv)
         );
 
-        Map<String, Supplier<DocIdSetIterator>> metricDocIdSetIterators = new LinkedHashMap<>();
+        Map<String, Supplier<StarTreeValuesIterator>> metricDocIdSetIterators = new LinkedHashMap<>();
 
         metricDocIdSetIterators.put(
             fullyQualifiedFieldNameForStarTreeMetricsDocValues(
@@ -3667,7 +3690,7 @@ private StarTreeValues getStarTreeValues(
                 "field2",
                 sf.getMetrics().get(0).getMetrics().get(0).getTypeName()
             ),
-            () -> m1sndv
+            () -> new SortedNumericStarTreeValuesIterator(m1sndv)
         );
         metricDocIdSetIterators.put(
             fullyQualifiedFieldNameForStarTreeMetricsDocValues(
@@ -3675,7 +3698,7 @@ private StarTreeValues getStarTreeValues(
                 "_doc_count",
                 sf.getMetrics().get(1).getMetrics().get(0).getTypeName()
             ),
-            () -> m2sndv
+            () -> new SortedNumericStarTreeValuesIterator(m2sndv)
         );
         // metricDocIdSetIterators.put("field2", () -> m1sndv);
         // metricDocIdSetIterators.put("_doc_count", () -> m2sndv);
@@ -4093,24 +4116,24 @@ public void testMergeFlow() throws IOException {
         SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField);
         SortedNumericDocValues valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount);
         SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric();
-        Map<String, Supplier<DocIdSetIterator>> dimDocIdSetIterators = Map.of(
+        Map<String, Supplier<StarTreeValuesIterator>> dimDocIdSetIterators = Map.of(
             "field1",
-            () -> d1sndv,
+            () -> new SortedNumericStarTreeValuesIterator(d1sndv),
             "field3",
-            () -> d2sndv,
+            () -> new SortedNumericStarTreeValuesIterator(d2sndv),
             "field5",
-            () -> d3sndv,
+            () -> new SortedNumericStarTreeValuesIterator(d3sndv),
             "field8",
-            () -> d4sndv
+            () -> new SortedNumericStarTreeValuesIterator(d4sndv)
         );
 
-        Map<String, Supplier<DocIdSetIterator>> metricDocIdSetIterators = Map.of(
+        Map<String, Supplier<StarTreeValuesIterator>> metricDocIdSetIterators = Map.of(
             "sf_field2_sum_metric",
-            () -> m1sndv,
+            () -> new SortedNumericStarTreeValuesIterator(m1sndv),
             "sf_field2_value_count_metric",
-            () -> valucountsndv,
+            () -> new SortedNumericStarTreeValuesIterator(valucountsndv),
             "sf__doc_count_doc_count_metric",
-            () -> m2sndv
+            () -> new SortedNumericStarTreeValuesIterator(m2sndv)
         );
 
         StarTreeValues starTreeValues = new StarTreeValues(
@@ -4129,24 +4152,24 @@ public void testMergeFlow() throws IOException {
         SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField);
         SortedNumericDocValues f2valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount);
         SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric();
-        Map<String, Supplier<DocIdSetIterator>> f2dimDocIdSetIterators = Map.of(
+        Map<String, Supplier<StarTreeValuesIterator>> f2dimDocIdSetIterators = Map.of(
             "field1",
-            () -> f2d1sndv,
+            () -> new SortedNumericStarTreeValuesIterator(f2d1sndv),
             "field3",
-            () -> f2d2sndv,
+            () -> new SortedNumericStarTreeValuesIterator(f2d2sndv),
             "field5",
-            () -> f2d3sndv,
+            () -> new SortedNumericStarTreeValuesIterator(f2d3sndv),
             "field8",
-            () -> f2d4sndv
+            () -> new SortedNumericStarTreeValuesIterator(f2d4sndv)
         );
 
-        Map<String, Supplier<DocIdSetIterator>> f2metricDocIdSetIterators = Map.of(
+        Map<String, Supplier<StarTreeValuesIterator>> f2metricDocIdSetIterators = Map.of(
             "sf_field2_sum_metric",
-            () -> f2m1sndv,
+            () -> new SortedNumericStarTreeValuesIterator(f2m1sndv),
             "sf_field2_value_count_metric",
-            () -> f2valucountsndv,
+            () -> new SortedNumericStarTreeValuesIterator(f2valucountsndv),
             "sf__doc_count_doc_count_metric",
-            () -> f2m2sndv
+            () -> new SortedNumericStarTreeValuesIterator(f2m2sndv)
         );
         StarTreeValues starTreeValues2 = new StarTreeValues(
             compositeField,
diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java
index f56f7d9906ae1..78d63800abd16 100644
--- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java
+++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java
@@ -9,15 +9,13 @@
 package org.opensearch.index.compositeindex.datacube.startree.utils;
 
 import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.util.BytesRef;
+import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.BeforeClass;
 
@@ -59,50 +57,32 @@ public void testCreateIterator_SortedNumeric() throws IOException {
         DocValuesProducer producer = Mockito.mock(DocValuesProducer.class);
         SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class);
         when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator);
-        SequentialDocValuesIterator result = new SequentialDocValuesIterator(producer.getSortedNumeric(mockFieldInfo));
-        assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass());
-    }
-
-    public void testCreateIterator_UnsupportedType() throws IOException {
-        DocValuesProducer producer = Mockito.mock(DocValuesProducer.class);
-        BinaryDocValues iterator = Mockito.mock(BinaryDocValues.class);
-        when(producer.getBinary(mockFieldInfo)).thenReturn(iterator);
-        SequentialDocValuesIterator result = new SequentialDocValuesIterator(producer.getBinary(mockFieldInfo));
-        assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass());
-        when(iterator.nextDoc()).thenReturn(0);
-        when(iterator.binaryValue()).thenReturn(new BytesRef("123"));
+        SequentialDocValuesIterator result = new SequentialDocValuesIterator(
+            new SortedNumericStarTreeValuesIterator(producer.getSortedNumeric(mockFieldInfo))
+        );
 
-        IllegalStateException exception = expectThrows(IllegalStateException.class, () -> {
-            result.nextDoc(0);
-            result.value(0);
-        });
-        assertEquals("Unsupported Iterator requested for SequentialDocValuesIterator", exception.getMessage());
     }
 
     public void testGetNextValue_SortedNumeric() throws IOException {
         SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class);
         when(iterator.nextDoc()).thenReturn(0);
         when(iterator.nextValue()).thenReturn(123L);
-        SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator);
-        sequentialDocValuesIterator.nextDoc(0);
+        SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(
+            new SortedNumericStarTreeValuesIterator(iterator)
+        );
+        sequentialDocValuesIterator.nextEntry(0);
         long result = sequentialDocValuesIterator.value(0);
         assertEquals(123L, result);
     }
 
-    public void testGetNextValue_UnsupportedIterator() {
-        DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class);
-        SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator);
-
-        IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { sequentialDocValuesIterator.value(0); });
-        assertEquals("Unsupported Iterator requested for SequentialDocValuesIterator", exception.getMessage());
-    }
-
-    public void testNextDoc() throws IOException {
+    public void testNextEntry() throws IOException {
         SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class);
-        SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator);
+        SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(
+            new SortedNumericStarTreeValuesIterator(iterator)
+        );
         when(iterator.nextDoc()).thenReturn(5);
 
-        int result = sequentialDocValuesIterator.nextDoc(5);
+        int result = sequentialDocValuesIterator.nextEntry(5);
         assertEquals(5, result);
     }
 
@@ -110,8 +90,12 @@ public void test_multipleCoordinatedDocumentReader() throws IOException {
         SortedNumericDocValues iterator1 = Mockito.mock(SortedNumericDocValues.class);
         SortedNumericDocValues iterator2 = Mockito.mock(SortedNumericDocValues.class);
 
-        SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator(iterator1);
-        SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator(iterator2);
+        SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator(
+            new SortedNumericStarTreeValuesIterator(iterator1)
+        );
+        SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator(
+            new SortedNumericStarTreeValuesIterator(iterator2)
+        );
 
         when(iterator1.nextDoc()).thenReturn(0);
         when(iterator2.nextDoc()).thenReturn(1);
@@ -119,13 +103,13 @@ public void test_multipleCoordinatedDocumentReader() throws IOException {
         when(iterator1.nextValue()).thenReturn(9L);
         when(iterator2.nextValue()).thenReturn(9L);
 
-        sequentialDocValuesIterator1.nextDoc(0);
-        sequentialDocValuesIterator2.nextDoc(0);
-        assertEquals(0, sequentialDocValuesIterator1.getDocId());
+        sequentialDocValuesIterator1.nextEntry(0);
+        sequentialDocValuesIterator2.nextEntry(0);
+        assertEquals(0, sequentialDocValuesIterator1.getEntryId());
         assertEquals(9L, (long) sequentialDocValuesIterator1.value(0));
         assertNull(sequentialDocValuesIterator2.value(0));
-        assertNotEquals(0, sequentialDocValuesIterator2.getDocId());
-        assertEquals(1, sequentialDocValuesIterator2.getDocId());
+        assertNotEquals(0, sequentialDocValuesIterator2.getEntryId());
+        assertEquals(1, sequentialDocValuesIterator2.getEntryId());
         assertEquals(9L, (long) sequentialDocValuesIterator2.value(1));
     }
 }
diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
index cb06bf23d9cbe..b415e1e657f7f 100644
--- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
@@ -51,8 +51,12 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Collections;
+
+import org.mockito.Mockito;
 
 import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX;
+import static org.opensearch.index.mapper.ObjectMapper.Nested.isParent;
 import static org.hamcrest.Matchers.containsString;
 
 public class ObjectMapperTests extends OpenSearchSingleNodeTestCase {
@@ -568,6 +572,49 @@ public void testCompositeFields() throws Exception {
         FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
     }
 
+    public void testNestedIsParent() throws Exception {
+        String mapping = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("properties")
+            .startObject("a")
+            .field("type", "nested")
+            .startObject("properties")
+            .field("b1", Collections.singletonMap("type", "keyword"))
+            .startObject("b2")
+            .field("type", "nested")
+            .startObject("properties")
+            .startObject("c")
+            .field("type", "nested")
+            .startObject("properties")
+            .field("d", Collections.singletonMap("type", "keyword"))
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .toString();
+
+        DocumentMapper documentMapper = createIndex("test").mapperService()
+            .documentMapperParser()
+            .parse("_doc", new CompressedXContent(mapping));
+
+        MapperService mapperService = Mockito.mock(MapperService.class);
+        Mockito.when(mapperService.getObjectMapper(("a"))).thenReturn(documentMapper.objectMappers().get("a"));
+        Mockito.when(mapperService.getObjectMapper(("a.b2"))).thenReturn(documentMapper.objectMappers().get("a.b2"));
+        Mockito.when(mapperService.getObjectMapper(("a.b2.c"))).thenReturn(documentMapper.objectMappers().get("a.b2.c"));
+
+        assertTrue(isParent(documentMapper.objectMappers().get("a"), documentMapper.objectMappers().get("a.b2.c"), mapperService));
+        assertTrue(isParent(documentMapper.objectMappers().get("a"), documentMapper.objectMappers().get("a.b2"), mapperService));
+        assertTrue(isParent(documentMapper.objectMappers().get("a.b2"), documentMapper.objectMappers().get("a.b2.c"), mapperService));
+
+        assertFalse(isParent(documentMapper.objectMappers().get("a.b2.c"), documentMapper.objectMappers().get("a"), mapperService));
+        assertFalse(isParent(documentMapper.objectMappers().get("a.b2"), documentMapper.objectMappers().get("a"), mapperService));
+        assertFalse(isParent(documentMapper.objectMappers().get("a.b2.c"), documentMapper.objectMappers().get("a.b2"), mapperService));
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java
index be30de97ee830..d134a2bad7b74 100644
--- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java
@@ -1083,4 +1083,101 @@ public void testIsPinnedTimestampStateStaleFeatureEnabled() {
         setupRemotePinnedTimestampFeature(true);
         assertTrue(RemoteStoreUtils.isPinnedTimestampStateStale());
     }
+
+    public void testGetPinnedTimestampLockedFilesWithCache() {
+        setupRemotePinnedTimestampFeature(true);
+
+        Map<Long, String> metadataFilePinnedTimestampCache = new HashMap<>();
+
+        // Pinned timestamps 800, 900, 1000, 2000
+        // Metadata with timestamp 990, 995, 1000, 1001
+        // Metadata timestamp 1000 <= Pinned Timestamp 1000
+        // Metadata timestamp 1001 <= Pinned Timestamp 2000
+        Tuple<Map<Long, String>, Set<String>> metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps(
+            List.of(990L, 995L, 1000L, 1001L),
+            Set.of(800L, 900L, 1000L, 2000L),
+            metadataFilePinnedTimestampCache
+        );
+        Map<Long, String> metadataFiles = metadataAndLocks.v1();
+        Set<String> implicitLockedFiles = metadataAndLocks.v2();
+
+        assertEquals(2, implicitLockedFiles.size());
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(1000L)));
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(1001L)));
+        // Now we cache all the matches except the last one.
+        assertEquals(1, metadataFilePinnedTimestampCache.size());
+        assertEquals(metadataFiles.get(1000L), metadataFilePinnedTimestampCache.get(1000L));
+
+        metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps(
+            List.of(990L, 995L, 1000L, 1001L, 2000L, 2200L),
+            Set.of(800L, 900L, 1000L, 2000L, 3000L),
+            metadataFilePinnedTimestampCache
+        );
+        metadataFiles = metadataAndLocks.v1();
+        implicitLockedFiles = metadataAndLocks.v2();
+        assertEquals(3, implicitLockedFiles.size());
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(1000L)));
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(2000L)));
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(2200L)));
+        assertEquals(2, metadataFilePinnedTimestampCache.size());
+        assertEquals(metadataFiles.get(1000L), metadataFilePinnedTimestampCache.get(1000L));
+        assertEquals(metadataFiles.get(2000L), metadataFilePinnedTimestampCache.get(2000L));
+
+        metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps(
+            List.of(990L, 995L, 1000L, 1001L, 2000L, 2200L, 2500L),
+            Set.of(2000L, 3000L),
+            metadataFilePinnedTimestampCache
+        );
+        metadataFiles = metadataAndLocks.v1();
+        implicitLockedFiles = metadataAndLocks.v2();
+        assertEquals(2, implicitLockedFiles.size());
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(2000L)));
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(2500L)));
+        assertEquals(1, metadataFilePinnedTimestampCache.size());
+        assertEquals(metadataFiles.get(2000L), metadataFilePinnedTimestampCache.get(2000L));
+
+        metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps(
+            List.of(2000L, 2200L, 2500L, 3001L, 4200L, 4600L, 5010L),
+            Set.of(3000L, 4000L, 5000L, 6000L),
+            metadataFilePinnedTimestampCache
+        );
+        metadataFiles = metadataAndLocks.v1();
+        implicitLockedFiles = metadataAndLocks.v2();
+        assertEquals(4, implicitLockedFiles.size());
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(2500L)));
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(3001L)));
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(4600L)));
+        assertTrue(implicitLockedFiles.contains(metadataFiles.get(5010L)));
+        assertEquals(3, metadataFilePinnedTimestampCache.size());
+        assertEquals(metadataFiles.get(2500L), metadataFilePinnedTimestampCache.get(3000L));
+        assertEquals(metadataFiles.get(3001L), metadataFilePinnedTimestampCache.get(4000L));
+        assertEquals(metadataFiles.get(4600L), metadataFilePinnedTimestampCache.get(5000L));
+
+        metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps(
+            List.of(),
+            Set.of(3000L, 4000L, 5000L, 6000L),
+            metadataFilePinnedTimestampCache
+        );
+        implicitLockedFiles = metadataAndLocks.v2();
+        assertEquals(0, implicitLockedFiles.size());
+        assertEquals(3, metadataFilePinnedTimestampCache.size());
+
+        assertThrows(
+            AssertionError.class,
+            () -> testGetPinnedTimestampLockedFilesWithPinnedTimestamps(
+                List.of(2000L, 2200L, 3001L, 4200L, 4600L, 5010L),
+                Set.of(3000L, 4000L, 5000L, 6000L),
+                metadataFilePinnedTimestampCache
+            )
+        );
+
+        metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps(
+            List.of(2000L, 2200L, 2500L, 3001L, 4200L, 4600L, 5010L),
+            Set.of(),
+            metadataFilePinnedTimestampCache
+        );
+        implicitLockedFiles = metadataAndLocks.v2();
+        assertEquals(0, implicitLockedFiles.size());
+        assertEquals(0, metadataFilePinnedTimestampCache.size());
+    }
 }
diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
index ecd6620dbea15..df3df81361a12 100644
--- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
+++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
@@ -565,7 +565,8 @@ public void testCleanupAsync() throws Exception {
             repositoryName,
             indexUUID,
             shardId,
-            pathStrategy
+            pathStrategy,
+            false
         );
         verify(remoteSegmentStoreDirectoryFactory).newDirectory(repositoryName, indexUUID, shardId, pathStrategy);
         verify(threadPool, times(0)).executor(ThreadPool.Names.REMOTE_PURGE);
diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java
index 4ec68f7fb75b4..73db3314f4d1e 100644
--- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java
@@ -214,6 +214,7 @@ public void onFailure(Exception e) {
         // Old format metadata file
         String oldFormatMdFilename = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1";
         assertNull(TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(oldFormatMdFilename));
+        assertEquals(Long.MAX_VALUE - 9223372036854774799L, TranslogTransferMetadata.getMaxGenerationFromFileName(oldFormatMdFilename));
 
         // Node id containing separator
         String nodeIdWithSeparator =
@@ -221,10 +222,14 @@ public void onFailure(Exception e) {
         Tuple<Long, Long> minMaxGen = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(nodeIdWithSeparator);
         Long minGen = Long.MAX_VALUE - 9223372036438563958L;
         assertEquals(minGen, minMaxGen.v1());
+        Long maxGen = Long.MAX_VALUE - 9223372036854774799L;
+        assertEquals(maxGen, minMaxGen.v2());
+        assertEquals(Long.MAX_VALUE - 9223372036854774799L, TranslogTransferMetadata.getMaxGenerationFromFileName(nodeIdWithSeparator));
 
         // Malformed md filename
         String malformedMdFileName = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__node1__xyz__3__1";
         assertNull(TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(malformedMdFileName));
+        assertEquals(Long.MAX_VALUE - 9223372036854774799L, TranslogTransferMetadata.getMaxGenerationFromFileName(malformedMdFileName));
     }
 
     public void testGetMinMaxPrimaryTermFromFilename() throws Exception {
@@ -330,43 +335,60 @@ public void testSimpleOperationsUpload() throws Exception {
 
         addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 }));
         addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 3, primaryTerm.get(), new byte[] { 1 }));
-        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 }));
-        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("5", 5, primaryTerm.get(), new byte[] { 1 }));
-        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("6", 6, primaryTerm.get(), new byte[] { 1 }));
 
         assertBusy(() -> {
             assertEquals(
-                16,
+                10,
                 blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size()
             );
         });
 
-        assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
-
         RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO);
+        assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
         // Fetch pinned timestamps so that it won't be stale
         updatePinnedTimstampTask.run();
+        translog.setMinSeqNoToKeep(3);
+        translog.trimUnreferencedReaders();
+
+        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 }));
+        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("5", 5, primaryTerm.get(), new byte[] { 1 }));
+        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("6", 6, primaryTerm.get(), new byte[] { 1 }));
 
-        translog.setMinSeqNoToKeep(4);
+        assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
+        // Fetch pinned timestamps so that it won't be stale
+        updatePinnedTimstampTask.run();
+        translog.setMinSeqNoToKeep(6);
         translog.trimUnreferencedReaders();
+        assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
+
+        assertEquals(1, translog.readers.size());
+        assertBusy(() -> {
+            assertEquals(2, translog.allUploaded().size());
+            assertEquals(4, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size());
+            assertEquals(
+                16,
+                blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size()
+            );
+        }, 30, TimeUnit.SECONDS);
+
         addToTranslogAndListAndUpload(translog, ops, new Translog.Index("7", 7, primaryTerm.get(), new byte[] { 1 }));
         addToTranslogAndListAndUpload(translog, ops, new Translog.Index("8", 8, primaryTerm.get(), new byte[] { 1 }));
-        assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
 
+        assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
         // Fetch pinned timestamps so that it won't be stale
         updatePinnedTimstampTask.run();
         translog.trimUnreferencedReaders();
-
         assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
-        assertEquals(5, translog.readers.size());
+
+        assertEquals(3, translog.readers.size());
         assertBusy(() -> {
-            assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size());
-            assertEquals(10, translog.allUploaded().size());
+            assertEquals(6, translog.allUploaded().size());
+            assertEquals(3, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size());
             assertEquals(
-                10,
+                12,
                 blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size()
             );
-        }, 60, TimeUnit.SECONDS);
+        }, 30, TimeUnit.SECONDS);
     }
 
     @Override
@@ -397,7 +419,7 @@ public void testMetadataFileDeletion() throws Exception {
         );
         updatePinnedTimstampTask.run();
         translog.trimUnreferencedReaders();
-        assertBusy(() -> { assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); });
+        assertBusy(() -> { assertEquals(3, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); });
     }
 
     public void testMetadataFileDeletionWithPinnedTimestamps() throws Exception {
@@ -568,7 +590,7 @@ public void testDrainSync() throws Exception {
         assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
         assertEquals(1, translog.readers.size());
         assertBusy(() -> assertEquals(2, translog.allUploaded().size()));
-        assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()));
+        assertBusy(() -> assertEquals(2, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()));
     }
 
     @Override
@@ -647,7 +669,7 @@ public void testGetGenerationsToBeDeletedEmptyMetadataFilesNotToBeDeleted() thro
         Set<Long> generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted(
             metadataFilesNotToBeDeleted,
             metadataFilesToBeDeleted,
-            true
+            Long.MAX_VALUE
         );
         Set<Long> md1Generations = LongStream.rangeClosed(4, 7).boxed().collect(Collectors.toSet());
         Set<Long> md2Generations = LongStream.rangeClosed(17, 37).boxed().collect(Collectors.toSet());
@@ -683,7 +705,7 @@ public void testGetGenerationsToBeDeleted() throws IOException {
         Set<Long> generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted(
             metadataFilesNotToBeDeleted,
             metadataFilesToBeDeleted,
-            true
+            Long.MAX_VALUE
         );
         Set<Long> md1Generations = LongStream.rangeClosed(5, 7).boxed().collect(Collectors.toSet());
         Set<Long> md2Generations = LongStream.rangeClosed(17, 25).boxed().collect(Collectors.toSet());
@@ -699,6 +721,42 @@ public void testGetGenerationsToBeDeleted() throws IOException {
         assertTrue(generations.isEmpty());
     }
 
+    public void testGetGenerationsToBeDeletedWithGenerationInRemote() throws IOException {
+        List<String> metadataFilesNotToBeDeleted = List.of(
+            // 1 to 4
+            "metadata__9223372036854775806__9223372036854775803__9223370311919910398__31__9223372036854775806__1__1",
+            // 26 to 30
+            "metadata__9223372036854775806__9223372036854775777__9223370311919910398__31__9223372036854775781__1__1",
+            // 42 to 100
+            "metadata__9223372036854775806__9223372036854775707__9223370311919910403__31__9223372036854775765__1__1"
+        );
+        List<String> metadataFilesToBeDeleted = List.of(
+            // 4 to 7
+            "metadata__9223372036854775806__9223372036854775800__9223370311919910398__31__9223372036854775803__1__1",
+            // 17 to 37
+            "metadata__9223372036854775806__9223372036854775770__9223370311919910398__31__9223372036854775790__1__1",
+            // 27 to 42
+            "metadata__9223372036854775806__9223372036854775765__9223370311919910403__31__9223372036854775780__1__1"
+        );
+        Set<Long> generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted(
+            metadataFilesNotToBeDeleted,
+            metadataFilesToBeDeleted,
+            35
+        );
+        Set<Long> md1Generations = LongStream.rangeClosed(5, 7).boxed().collect(Collectors.toSet());
+        Set<Long> md2Generations = LongStream.rangeClosed(17, 25).boxed().collect(Collectors.toSet());
+        Set<Long> md3Generations = LongStream.rangeClosed(31, 34).boxed().collect(Collectors.toSet());
+
+        assertTrue(generations.containsAll(md1Generations));
+        assertTrue(generations.containsAll(md2Generations));
+        assertTrue(generations.containsAll(md3Generations));
+
+        generations.removeAll(md1Generations);
+        generations.removeAll(md2Generations);
+        generations.removeAll(md3Generations);
+        assertTrue(generations.isEmpty());
+    }
+
     public void testGetMetadataFilesToBeDeletedNoExclusion() {
         updatePinnedTimstampTask.run();
 
@@ -708,7 +766,10 @@ public void testGetMetadataFilesToBeDeletedNoExclusion() {
             "metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1"
         );
 
-        assertEquals(metadataFiles, ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles));
+        assertEquals(
+            metadataFiles,
+            RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(metadataFiles, new HashMap<>(), Long.MAX_VALUE, false, logger)
+        );
     }
 
     public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() {
@@ -724,7 +785,13 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() {
             "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1"
         );
 
-        List<String> metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles);
+        List<String> metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(
+            metadataFiles,
+            new HashMap<>(),
+            Long.MAX_VALUE,
+            false,
+            logger
+        );
         assertEquals(1, metadataFilesToBeDeleted.size());
         assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0));
     }
@@ -746,7 +813,13 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnPinningOnly() throws
             "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1"
         );
 
-        List<String> metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles);
+        List<String> metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(
+            metadataFiles,
+            new HashMap<>(),
+            Long.MAX_VALUE,
+            false,
+            logger
+        );
         assertEquals(2, metadataFilesToBeDeleted.size());
         assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0));
         assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1));
@@ -769,11 +842,77 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeAndPinning() throw
             "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1"
         );
 
-        List<String> metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles);
+        List<String> metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(
+            metadataFiles,
+            new HashMap<>(),
+            Long.MAX_VALUE,
+            false,
+            logger
+        );
         assertEquals(1, metadataFilesToBeDeleted.size());
         assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(0));
     }
 
+    public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationOnly() throws IOException {
+        long currentTimeInMillis = System.currentTimeMillis();
+        String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000);
+        String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000);
+        String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000);
+
+        when(blobContainer.listBlobs()).thenReturn(Map.of());
+
+        updatePinnedTimstampTask.run();
+
+        List<String> metadataFiles = List.of(
+            // MaxGen 7
+            "metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1",
+            // MaxGen 12
+            "metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1",
+            // MaxGen 10
+            "metadata__9223372036438563903__9223372036854775798__" + md3Timestamp + "__31__9223372036854775701__1"
+        );
+
+        List<String> metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(
+            metadataFiles,
+            new HashMap<>(),
+            10L,
+            false,
+            logger
+        );
+        assertEquals(2, metadataFilesToBeDeleted.size());
+        assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0));
+        assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1));
+    }
+
+    public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationDeleteIndex() throws IOException {
+        long currentTimeInMillis = System.currentTimeMillis();
+        String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000);
+        String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000);
+        String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000);
+
+        when(blobContainer.listBlobs()).thenReturn(Map.of());
+
+        updatePinnedTimstampTask.run();
+
+        List<String> metadataFiles = List.of(
+            // MaxGen 7
+            "metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1",
+            // MaxGen 12
+            "metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1",
+            // MaxGen 17
+            "metadata__9223372036438563903__9223372036854775790__" + md3Timestamp + "__31__9223372036854775701__1"
+        );
+
+        List<String> metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(
+            metadataFiles,
+            new HashMap<>(),
+            10L,
+            true,
+            logger
+        );
+        assertEquals(metadataFiles, metadataFilesToBeDeleted);
+    }
+
     public void testIsGenerationPinned() {
         TreeSet<Tuple<Long, Long>> pinnedGenerations = new TreeSet<>(new TreeSet<>((o1, o2) -> {
             if (Objects.equals(o1.v1(), o2.v1()) == false) {
@@ -879,6 +1018,16 @@ public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException
                 translogTransferManager
             )
         );
+        // Calling it again to check if the details are getting fetched from the cache.
+        // Number of calls to readMetadata will be able to verify this.
+        assertEquals(
+            new Tuple<>(701L, 1008L),
+            translog.getMinMaxTranslogGenerationFromMetadataFile(
+                "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1",
+                translogTransferManager
+            )
+        );
+
         TranslogTransferMetadata md2 = mock(TranslogTransferMetadata.class);
         when(md2.getMinTranslogGeneration()).thenReturn(4L);
         when(md2.getGeneration()).thenReturn(7L);
@@ -891,9 +1040,81 @@ public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException
                 translogTransferManager
             )
         );
+        // Calling it again to check if the details are getting fetched from the cache.
+        // Number of calls to readMetadata will be able to verify this.
+        assertEquals(
+            new Tuple<>(4L, 7L),
+            translog.getMinMaxTranslogGenerationFromMetadataFile(
+                "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1",
+                translogTransferManager
+            )
+        );
 
-        verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1");
-        verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1");
+        verify(translogTransferManager, times(1)).readMetadata(
+            "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1"
+        );
+        verify(translogTransferManager, times(1)).readMetadata(
+            "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1"
+        );
+    }
+
+    public void testGetMinMaxPrimaryTermFromMetadataFile() throws IOException {
+        TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class);
+
+        RemoteFsTimestampAwareTranslog translog = (RemoteFsTimestampAwareTranslog) this.translog;
+
+        // Fetch generations directly from the filename
+        assertEquals(
+            new Tuple<>(1L, 1008L),
+            RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile(
+                "metadata__9223372036854774799__9223372036854774799__9223370311919910393__31__9223372036854775106__1__1",
+                translogTransferManager,
+                new HashMap<>()
+            )
+        );
+        assertEquals(
+            new Tuple<>(4L, 7L),
+            RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile(
+                "metadata__9223372036854775800__9223372036854775800__9223370311919910398__31__9223372036854775803__4__1",
+                translogTransferManager,
+                new HashMap<>()
+            )
+        );
+        assertEquals(
+            new Tuple<>(10L, 10L),
+            RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile(
+                "metadata__9223372036854775797__9223372036854775800__9223370311919910398__31__9223372036854775803__10__1",
+                translogTransferManager,
+                new HashMap<>()
+            )
+        );
+
+        // For older md filenames, it needs to read the content
+        TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class);
+        when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "1", "23", "1", "34", "2"));
+        when(translogTransferManager.readMetadata("metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1"))
+            .thenReturn(md1);
+        assertEquals(
+            new Tuple<>(1L, 2L),
+            RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile(
+                "metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1",
+                translogTransferManager,
+                new HashMap<>()
+            )
+        );
+        assertEquals(
+            new Tuple<>(4L, 7L),
+            RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile(
+                "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1",
+                translogTransferManager,
+                Map.of("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", new Tuple<>(4L, 7L))
+            )
+        );
+
+        verify(translogTransferManager).readMetadata("metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1");
+        verify(translogTransferManager, times(0)).readMetadata(
+            "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1"
+        );
     }
 
     public void testDeleteStaleRemotePrimaryTerms() throws IOException {
@@ -932,6 +1153,96 @@ public void testDeleteStaleRemotePrimaryTerms() throws IOException {
         verify(translogTransferManager, times(1)).deletePrimaryTermsAsync(anyLong());
     }
 
+    public void testDeleteStaleRemotePrimaryTermsOldFormat() throws IOException {
+        TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class);
+
+        List<String> metadataFiles = List.of(
+            // PT 4 to 9
+            "metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1",
+            // PT 2 to 7
+            "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1",
+            // PT 2 to 6
+            "metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1"
+        );
+
+        Logger staticLogger = LogManager.getLogger(RemoteFsTimestampAwareTranslogTests.class);
+        when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(1L, 2L, 3L, 4L));
+
+        TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class);
+        when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "4", "23", "5", "34", "5"));
+        when(translogTransferManager.readMetadata("metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1"))
+            .thenReturn(md1);
+        TranslogTransferMetadata md2 = mock(TranslogTransferMetadata.class);
+        when(md2.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "3"));
+        when(translogTransferManager.readMetadata("metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1"))
+            .thenReturn(md2);
+        TranslogTransferMetadata md3 = mock(TranslogTransferMetadata.class);
+        when(md3.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "2"));
+        when(translogTransferManager.readMetadata("metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1"))
+            .thenReturn(md3);
+
+        AtomicLong minPrimaryTermInRemote = new AtomicLong(Long.MAX_VALUE);
+        RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms(
+            metadataFiles,
+            translogTransferManager,
+            new HashMap<>(),
+            minPrimaryTermInRemote,
+            staticLogger
+        );
+        verify(translogTransferManager).deletePrimaryTermsAsync(2L);
+        assertEquals(2, minPrimaryTermInRemote.get());
+
+        RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms(
+            metadataFiles,
+            translogTransferManager,
+            new HashMap<>(),
+            minPrimaryTermInRemote,
+            staticLogger
+        );
+        // This means there are no new invocations of deletePrimaryTermAsync
+        verify(translogTransferManager, times(1)).deletePrimaryTermsAsync(anyLong());
+    }
+
+    public void testDeleteStaleRemotePrimaryTermsOldFormatException() throws IOException {
+        TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class);
+
+        List<String> metadataFiles = List.of(
+            // PT 4 to 9
+            "metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1",
+            // PT 2 to 7
+            "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1",
+            // PT 2 to 6
+            "metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1"
+        );
+
+        Logger staticLogger = LogManager.getLogger(RemoteFsTimestampAwareTranslogTests.class);
+        when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(1L, 2L, 3L, 4L));
+
+        TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class);
+        when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "4", "23", "5", "34", "5"));
+        when(translogTransferManager.readMetadata("metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1"))
+            .thenReturn(md1);
+        TranslogTransferMetadata md2 = mock(TranslogTransferMetadata.class);
+        when(md2.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "3"));
+        when(translogTransferManager.readMetadata("metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1"))
+            .thenReturn(md2);
+        TranslogTransferMetadata md3 = mock(TranslogTransferMetadata.class);
+        when(md3.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "2"));
+        // Exception while reading this file
+        when(translogTransferManager.readMetadata("metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1"))
+            .thenThrow(new IOException());
+
+        AtomicLong minPrimaryTermInRemote = new AtomicLong(4);
+        RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms(
+            metadataFiles,
+            translogTransferManager,
+            new HashMap<>(),
+            minPrimaryTermInRemote,
+            staticLogger
+        );
+        verify(translogTransferManager, times(0)).deletePrimaryTermsAsync(anyLong());
+    }
+
     public void testDeleteStaleRemotePrimaryTermsNoPrimaryTermInRemote() throws IOException {
         TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class);
 
@@ -984,4 +1295,41 @@ public void testDeleteStaleRemotePrimaryTermsPrimaryTermInRemoteIsBigger() throw
         assertEquals(2, minPrimaryTermInRemote.get());
     }
 
+    public void testGetMinPrimaryTermInRemoteAlreadyFetched() {
+        Long expectedMinPrimaryTerm = 12L;
+        assertEquals(expectedMinPrimaryTerm, RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(12), null, null));
+    }
+
+    public void testGetMinPrimaryTermInRemoteNotFetchedEmpty() throws IOException {
+        TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class);
+        when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of());
+        Long expectedMinPrimaryTerm = Long.MAX_VALUE;
+        assertEquals(
+            expectedMinPrimaryTerm,
+            RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(Long.MAX_VALUE), translogTransferManager, null)
+        );
+        verify(translogTransferManager).listPrimaryTermsInRemote();
+    }
+
+    public void testGetMinPrimaryTermInRemoteNotFetchedException() throws IOException {
+        TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class);
+        when(translogTransferManager.listPrimaryTermsInRemote()).thenThrow(new IOException());
+        Long expectedMinPrimaryTerm = Long.MAX_VALUE;
+        assertEquals(
+            expectedMinPrimaryTerm,
+            RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(Long.MAX_VALUE), translogTransferManager, logger)
+        );
+        verify(translogTransferManager).listPrimaryTermsInRemote();
+    }
+
+    public void testGetMinPrimaryTermInRemoteNotFetched() throws IOException {
+        TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class);
+        when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(12L, 23L, 34L, 4L, 89L));
+        Long expectedMinPrimaryTerm = 4L;
+        assertEquals(
+            expectedMinPrimaryTerm,
+            RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(Long.MAX_VALUE), translogTransferManager, logger)
+        );
+        verify(translogTransferManager).listPrimaryTermsInRemote();
+    }
 }
diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
index 339d876274557..03c77a9a83f57 100644
--- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
@@ -1801,6 +1801,83 @@ public void testDownloadWithEmptyTranslogOnlyInLocal() throws IOException {
         assertArrayEquals(filesPostFirstDownload, filesPostSecondDownload);
     }
 
+    public void testSyncWithGlobalCheckpointUpdate() throws IOException {
+        ArrayList<Translog.Operation> ops = new ArrayList<>();
+        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 }));
+        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 1, primaryTerm.get(), new byte[] { 2 }));
+
+        // Set a global checkpoint
+        long initialGlobalCheckpoint = 1L;
+        globalCheckpoint.set(initialGlobalCheckpoint);
+
+        // Sync the translog
+        translog.sync();
+
+        // Verify that the globalCheckpointSynced is updated
+        assertEquals(initialGlobalCheckpoint, ((RemoteFsTranslog) translog).getLastSyncedCheckpoint().globalCheckpoint);
+
+        // Update global checkpoint
+        long newGlobalCheckpoint = 2L;
+        globalCheckpoint.set(newGlobalCheckpoint);
+
+        // Add a new operation and sync
+        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 2, primaryTerm.get(), new byte[] { 3 }));
+        translog.sync();
+
+        // Verify that the globalCheckpointSynced is updated to the new value
+        assertEquals(newGlobalCheckpoint, ((RemoteFsTranslog) translog).getLastSyncedCheckpoint().globalCheckpoint);
+    }
+
+    public void testSyncNeededWithGlobalCheckpointUpdate() throws IOException {
+        ArrayList<Translog.Operation> ops = new ArrayList<>();
+        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 }));
+
+        // Set initial global checkpoint
+        long initialGlobalCheckpoint = 0L;
+        globalCheckpoint.set(initialGlobalCheckpoint);
+
+        // Sync the translog
+        translog.sync();
+
+        // Verify that sync is not needed
+        assertFalse(translog.syncNeeded());
+
+        // Update global checkpoint
+        long newGlobalCheckpoint = 1L;
+        globalCheckpoint.set(newGlobalCheckpoint);
+
+        // Verify that sync is now needed due to global checkpoint update
+        assertTrue(translog.syncNeeded());
+
+        // Sync again
+        translog.sync();
+
+        // Verify that sync is not needed after syncing
+        assertFalse(translog.syncNeeded());
+    }
+
+    public void testGlobalCheckpointUpdateDuringClose() throws IOException {
+        ArrayList<Translog.Operation> ops = new ArrayList<>();
+        addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 }));
+
+        // Set initial global checkpoint
+        long initialGlobalCheckpoint = 0L;
+        globalCheckpoint.set(initialGlobalCheckpoint);
+
+        // Sync the translog
+        translog.sync();
+
+        // Update global checkpoint
+        long newGlobalCheckpoint = 1L;
+        globalCheckpoint.set(newGlobalCheckpoint);
+
+        // Close the translog
+        translog.close();
+
+        // Verify that the last synced checkpoint includes the updated global checkpoint
+        assertEquals(newGlobalCheckpoint, ((RemoteFsTranslog) translog).getLastSyncedCheckpoint().globalCheckpoint);
+    }
+
     public class ThrowingBlobRepository extends FsRepository {
 
         private final Environment environment;
diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java
index 10e4cc6cfb1ef..fddc6c0c94005 100644
--- a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java
@@ -17,6 +17,7 @@
 import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.blobstore.BlobStore;
+import org.opensearch.common.blobstore.DeleteResult;
 import org.opensearch.common.blobstore.fs.FsBlobContainer;
 import org.opensearch.common.blobstore.fs.FsBlobStore;
 import org.opensearch.common.blobstore.stream.read.ReadContext;
@@ -51,6 +52,7 @@
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.Base64;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -340,5 +342,15 @@ public boolean remoteIntegrityCheckSupported() {
         public BlobContainer getDelegate() {
             return delegate;
         }
+
+        @Override
+        public void deleteBlobsAsyncIgnoringIfNotExists(List<String> blobNames, ActionListener<Void> completionListener) {
+            throw new RuntimeException("deleteBlobsAsyncIgnoringIfNotExists not supported");
+        }
+
+        @Override
+        public void deleteAsync(ActionListener<DeleteResult> completionListener) {
+            throw new RuntimeException("deleteAsync not supported");
+        }
     }
 }
diff --git a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java
index 45653e9d8e4d6..7534dcd93944a 100644
--- a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java
+++ b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java
@@ -35,6 +35,8 @@
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.common.Table;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.rest.RestStatus;
 import org.opensearch.rest.RestHandler.ReplacedRoute;
 import org.opensearch.rest.RestHandler.Route;
 import org.opensearch.rest.RestRequest.Method;
@@ -46,15 +48,22 @@
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
+import java.lang.reflect.Modifier;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.core.StringContains.containsString;
 import static org.hamcrest.object.HasToString.hasToString;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 
 public class BaseRestHandlerTests extends OpenSearchTestCase {
     private NodeClient mockClient;
@@ -288,4 +297,36 @@ public void testReplaceRoutesMethod() throws Exception {
         }
     }
 
+    public void testRestHandlerWrapper() throws Exception {
+        RestHandler rh = new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
+            }
+        };
+        RestHandler handlerSpy = spy(rh);
+        RestHandler.Wrapper rhWrapper = new RestHandler.Wrapper(handlerSpy);
+
+        List<java.lang.reflect.Method> overridableMethods = Arrays.stream(RestHandler.class.getMethods())
+            .filter(
+                m -> !(Modifier.isPrivate(m.getModifiers()) || Modifier.isStatic(m.getModifiers()) || Modifier.isFinal(m.getModifiers()))
+            )
+            .collect(Collectors.toList());
+
+        for (java.lang.reflect.Method method : overridableMethods) {
+            int argCount = method.getParameterCount();
+            Object[] args = new Object[argCount];
+            for (int i = 0; i < argCount; i++) {
+                args[i] = any();
+            }
+            if (args.length > 0) {
+                method.invoke(rhWrapper, args);
+            } else {
+                method.invoke(rhWrapper);
+            }
+            method.invoke(verify(handlerSpy, times(1)), args);
+        }
+        verifyNoMoreInteractions(handlerSpy);
+    }
+
 }
diff --git a/server/src/test/java/org/opensearch/rest/RestControllerTests.java b/server/src/test/java/org/opensearch/rest/RestControllerTests.java
index ef9257d746573..f7f1b02847854 100644
--- a/server/src/test/java/org/opensearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/opensearch/rest/RestControllerTests.java
@@ -55,13 +55,11 @@
 import org.opensearch.http.HttpResponse;
 import org.opensearch.http.HttpServerTransport;
 import org.opensearch.http.HttpStats;
-import org.opensearch.identity.IdentityService;
 import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.opensearch.rest.action.admin.indices.RestCreateIndexAction;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.client.NoOpNodeClient;
 import org.opensearch.test.rest.FakeRestRequest;
-import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.usage.UsageService;
 import org.junit.After;
 import org.junit.Before;
@@ -97,7 +95,6 @@ public class RestControllerTests extends OpenSearchTestCase {
     private RestController restController;
     private HierarchyCircuitBreakerService circuitBreakerService;
     private UsageService usageService;
-    private IdentityService identityService;
     private NodeClient client;
 
     @Before
@@ -115,11 +112,9 @@ public void setup() {
         // we can do this here only because we know that we don't adjust breaker settings dynamically in the test
         inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
 
-        identityService = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), List.of());
-
         HttpServerTransport httpServerTransport = new TestHttpServerTransport();
         client = new NoOpNodeClient(this.getTestName());
-        restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService, identityService);
+        restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService);
         restController.registerHandler(
             RestRequest.Method.GET,
             "/",
@@ -140,7 +135,7 @@ public void teardown() throws IOException {
     }
 
     public void testDefaultRestControllerGetAllHandlersContainsFavicon() {
-        final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService);
+        final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService);
         Iterator<MethodHandlers> handlers = restController.getAllHandlers();
         assertTrue(handlers.hasNext());
         MethodHandlers faviconHandler = handlers.next();
@@ -150,7 +145,7 @@ public void testDefaultRestControllerGetAllHandlersContainsFavicon() {
     }
 
     public void testRestControllerGetAllHandlers() {
-        final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService);
+        final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService);
 
         restController.registerHandler(RestRequest.Method.PATCH, "/foo", mock(RestHandler.class));
         restController.registerHandler(RestRequest.Method.GET, "/foo", mock(RestHandler.class));
@@ -175,7 +170,7 @@ public void testApplyRelevantHeaders() throws Exception {
         Set<RestHeaderDefinition> headers = new HashSet<>(
             Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", true))
         );
-        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, identityService);
+        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("true"));
         restHeaders.put("header.2", Collections.singletonList("true"));
@@ -211,7 +206,7 @@ public void testRequestWithDisallowedMultiValuedHeader() {
         Set<RestHeaderDefinition> headers = new HashSet<>(
             Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", false))
         );
-        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, identityService);
+        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("boo"));
         restHeaders.put("header.2", Arrays.asList("foo", "bar"));
@@ -226,14 +221,7 @@ public void testRequestWithDisallowedMultiValuedHeaderButSameValues() {
         Set<RestHeaderDefinition> headers = new HashSet<>(
             Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", false))
         );
-        final RestController restController = new RestController(
-            headers,
-            null,
-            client,
-            circuitBreakerService,
-            usageService,
-            identityService
-        );
+        final RestController restController = new RestController(headers, null, client, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("boo"));
         restHeaders.put("header.2", Arrays.asList("foo", "foo"));
@@ -294,7 +282,7 @@ public void testRegisterWithDeprecatedHandler() {
     }
 
     public void testRegisterSecondMethodWithDifferentNamedWildcard() {
-        final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService);
+        final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService);
 
         RestRequest.Method firstMethod = randomFrom(RestRequest.Method.values());
         RestRequest.Method secondMethod = randomFrom(
@@ -322,7 +310,7 @@ public void testRestHandlerWrapper() throws Exception {
         final RestController restController = new RestController(Collections.emptySet(), h -> {
             assertSame(handler, h);
             return (RestRequest request, RestChannel channel, NodeClient client) -> wrapperCalled.set(true);
-        }, client, circuitBreakerService, usageService, identityService);
+        }, client, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/wrapped", handler);
         RestRequest request = testRestRequest("/wrapped", "{}", MediaTypeRegistry.JSON);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST);
@@ -385,7 +373,7 @@ public void testDispatchRequiresContentTypeForRequestsWithContent() {
         String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead()));
         RestRequest request = testRestRequest("/", content, null);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE);
-        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService, identityService);
+        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService);
         restController.registerHandler(
             RestRequest.Method.GET,
             "/",
diff --git a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java
index 983121a4f481d..b8602cdc20e6a 100644
--- a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java
+++ b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java
@@ -39,12 +39,10 @@
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.indices.breaker.CircuitBreakerService;
 import org.opensearch.core.rest.RestStatus;
-import org.opensearch.identity.IdentityService;
 import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.rest.FakeRestChannel;
 import org.opensearch.test.rest.FakeRestRequest;
-import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.usage.UsageService;
 
 import java.util.ArrayList;
@@ -56,7 +54,6 @@
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.is;
-import static org.mockito.Mockito.mock;
 
 public class RestHttpResponseHeadersTests extends OpenSearchTestCase {
 
@@ -106,17 +103,8 @@ public void testUnsupportedMethodResponseHttpHeader() throws Exception {
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
         );
 
-        final Settings settings = Settings.EMPTY;
         UsageService usageService = new UsageService();
-        final IdentityService identityService = new IdentityService(settings, mock(ThreadPool.class), List.of());
-        RestController restController = new RestController(
-            Collections.emptySet(),
-            null,
-            null,
-            circuitBreakerService,
-            usageService,
-            identityService
-        );
+        RestController restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService);
 
         // A basic RestHandler handles requests to the endpoint
         RestHandler restHandler = new RestHandler() {
diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java
index c3cf33f4e9034..6aa1d10d71e50 100644
--- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java
+++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java
@@ -44,7 +44,6 @@
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
-import org.opensearch.identity.IdentityService;
 import org.opensearch.rest.RestController;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.search.AbstractSearchTestCase;
@@ -61,7 +60,6 @@
 
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
@@ -75,15 +73,7 @@ public class RestValidateQueryActionTests extends AbstractSearchTestCase {
     private static NodeClient client = new NodeClient(Settings.EMPTY, threadPool);
 
     private static UsageService usageService = new UsageService();
-    private static IdentityService identityService = new IdentityService(Settings.EMPTY, threadPool, List.of());
-    private static RestController controller = new RestController(
-        emptySet(),
-        null,
-        client,
-        new NoneCircuitBreakerService(),
-        usageService,
-        identityService
-    );
+    private static RestController controller = new RestController(emptySet(), null, client, new NoneCircuitBreakerService(), usageService);
     private static RestValidateQueryAction action = new RestValidateQueryAction();
 
     /**
diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java
index 96b1c75371697..2debb00a0d14f 100644
--- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java
+++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java
@@ -43,13 +43,19 @@
 import org.opensearch.cluster.routing.TestShardRouting;
 import org.opensearch.common.Table;
 import org.opensearch.common.UUIDs;
+import org.opensearch.common.breaker.ResponseLimitSettings;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.IndexSettings;
+import org.opensearch.rest.action.list.RestIndicesListAction;
+import org.opensearch.rest.pagination.PageToken;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.rest.FakeRestRequest;
+import org.junit.Before;
 
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -63,13 +69,14 @@
 
 public class RestIndicesActionTests extends OpenSearchTestCase {
 
-    public void testBuildTable() {
-        final int numIndices = randomIntBetween(3, 20);
-        final Map<String, Settings> indicesSettings = new LinkedHashMap<>();
-        final Map<String, IndexMetadata> indicesMetadatas = new LinkedHashMap<>();
-        final Map<String, ClusterIndexHealth> indicesHealths = new LinkedHashMap<>();
-        final Map<String, IndexStats> indicesStats = new LinkedHashMap<>();
+    final Map<String, Settings> indicesSettings = new LinkedHashMap<>();
+    final Map<String, IndexMetadata> indicesMetadatas = new LinkedHashMap<>();
+    final Map<String, ClusterIndexHealth> indicesHealths = new LinkedHashMap<>();
+    final Map<String, IndexStats> indicesStats = new LinkedHashMap<>();
 
+    @Before
+    public void setup() {
+        final int numIndices = randomIntBetween(3, 20);
         for (int i = 0; i < numIndices; i++) {
             String indexName = "index-" + i;
 
@@ -136,11 +143,65 @@ public void testBuildTable() {
                 }
             }
         }
+    }
 
-        final RestIndicesAction action = new RestIndicesAction();
-        final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetadatas);
+    public void testBuildTable() {
+        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final Settings settings = Settings.builder().build();
+        final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings);
+        final RestIndicesAction action = new RestIndicesAction(responseLimitSettings);
+        final Table table = action.buildTable(
+            new FakeRestRequest(),
+            indicesSettings,
+            indicesHealths,
+            indicesStats,
+            indicesMetadatas,
+            action.getTableIterator(new String[0], indicesSettings),
+            null
+        );
 
         // now, verify the table is correct
+        assertNotNull(table);
+
+        assertTableHeaders(table);
+
+        assertThat(table.getRows().size(), equalTo(indicesMetadatas.size()));
+        assertTableRows(table);
+    }
+
+    public void testBuildPaginatedTable() {
+        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final Settings settings = Settings.builder().build();
+        final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings);
+        final RestIndicesAction action = new RestIndicesAction(responseLimitSettings);
+        final RestIndicesListAction indicesListAction = new RestIndicesListAction(responseLimitSettings);
+        List<String> indicesList = new ArrayList<>(indicesMetadatas.keySet());
+        // Using half of the indices from metadata list for a page
+        String[] indicesToBeQueried = indicesList.subList(0, indicesMetadatas.size() / 2).toArray(new String[0]);
+        PageToken pageToken = new PageToken("foo", "indices");
+        final Table table = action.buildTable(
+            new FakeRestRequest(),
+            indicesSettings,
+            indicesHealths,
+            indicesStats,
+            indicesMetadatas,
+            indicesListAction.getTableIterator(indicesToBeQueried, indicesSettings),
+            pageToken
+        );
+
+        // verifying table
+        assertNotNull(table);
+        assertTableHeaders(table);
+        assertNotNull(table.getPageToken());
+        assertEquals(pageToken.getNextToken(), table.getPageToken().getNextToken());
+        assertEquals(pageToken.getPaginatedEntity(), table.getPageToken().getPaginatedEntity());
+
+        // Table should only contain the indices present in indicesToBeQueried
+        assertThat(table.getRows().size(), equalTo(indicesMetadatas.size() / 2));
+        assertTableRows(table);
+    }
+
+    private void assertTableHeaders(Table table) {
         List<Table.Cell> headers = table.getHeaders();
         assertThat(headers.get(0).value, equalTo("health"));
         assertThat(headers.get(1).value, equalTo("status"));
@@ -148,9 +209,10 @@ public void testBuildTable() {
         assertThat(headers.get(3).value, equalTo("uuid"));
         assertThat(headers.get(4).value, equalTo("pri"));
         assertThat(headers.get(5).value, equalTo("rep"));
+    }
 
+    private void assertTableRows(Table table) {
         final List<List<Table.Cell>> rows = table.getRows();
-        assertThat(rows.size(), equalTo(indicesMetadatas.size()));
 
         for (final List<Table.Cell> row : rows) {
             final String indexName = (String) row.get(2).value;
diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java
index 8183cb1d3b910..a82e563d70273 100644
--- a/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java
+++ b/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java
@@ -37,6 +37,7 @@
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.rest.AbstractRestChannel;
 import org.opensearch.rest.RestResponse;
+import org.opensearch.rest.pagination.PageToken;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.rest.FakeRestRequest;
 import org.junit.Before;
@@ -64,9 +65,14 @@ public class RestTableTests extends OpenSearchTestCase {
     private static final String ACCEPT = "Accept";
     private static final String TEXT_PLAIN = "text/plain; charset=UTF-8";
     private static final String TEXT_TABLE_BODY = "foo foo foo foo foo foo foo foo\n";
+    private static final String PAGINATED_TEXT_TABLE_BODY = "foo foo foo foo foo foo foo foo\nnext_token foo\n";
     private static final String JSON_TABLE_BODY = "[{\"bulk.foo\":\"foo\",\"bulk.bar\":\"foo\",\"aliasedBulk\":\"foo\","
         + "\"aliasedSecondBulk\":\"foo\",\"unmatched\":\"foo\","
         + "\"invalidAliasesBulk\":\"foo\",\"timestamp\":\"foo\",\"epoch\":\"foo\"}]";
+    private static final String PAGINATED_JSON_TABLE_BODY =
+        "{\"next_token\":\"foo\",\"entities\":[{\"bulk.foo\":\"foo\",\"bulk.bar\":\"foo\",\"aliasedBulk\":\"foo\","
+            + "\"aliasedSecondBulk\":\"foo\",\"unmatched\":\"foo\","
+            + "\"invalidAliasesBulk\":\"foo\",\"timestamp\":\"foo\",\"epoch\":\"foo\"}]}";
     private static final String YAML_TABLE_BODY = "---\n"
         + "- bulk.foo: \"foo\"\n"
         + "  bulk.bar: \"foo\"\n"
@@ -76,6 +82,17 @@ public class RestTableTests extends OpenSearchTestCase {
         + "  invalidAliasesBulk: \"foo\"\n"
         + "  timestamp: \"foo\"\n"
         + "  epoch: \"foo\"\n";
+    private static final String PAGINATED_YAML_TABLE_BODY = "---\n"
+        + "next_token: \"foo\"\n"
+        + "entities:\n"
+        + "- bulk.foo: \"foo\"\n"
+        + "  bulk.bar: \"foo\"\n"
+        + "  aliasedBulk: \"foo\"\n"
+        + "  aliasedSecondBulk: \"foo\"\n"
+        + "  unmatched: \"foo\"\n"
+        + "  invalidAliasesBulk: \"foo\"\n"
+        + "  timestamp: \"foo\"\n"
+        + "  epoch: \"foo\"\n";
     private Table table;
     private FakeRestRequest restRequest;
 
@@ -83,20 +100,7 @@ public class RestTableTests extends OpenSearchTestCase {
     public void setup() {
         restRequest = new FakeRestRequest();
         table = new Table();
-        table.startHeaders();
-        table.addCell("bulk.foo", "alias:f;desc:foo");
-        table.addCell("bulk.bar", "alias:b;desc:bar");
-        // should be matched as well due to the aliases
-        table.addCell("aliasedBulk", "alias:bulkWhatever;desc:bar");
-        table.addCell("aliasedSecondBulk", "alias:foobar,bulkolicious,bulkotastic;desc:bar");
-        // no match
-        table.addCell("unmatched", "alias:un.matched;desc:bar");
-        // invalid alias
-        table.addCell("invalidAliasesBulk", "alias:,,,;desc:bar");
-        // timestamp
-        table.addCell("timestamp", "alias:ts");
-        table.addCell("epoch", "alias:t");
-        table.endHeaders();
+        addHeaders(table);
     }
 
     public void testThatDisplayHeadersSupportWildcards() throws Exception {
@@ -121,10 +125,28 @@ public void testThatWeUseTheAcceptHeaderJson() throws Exception {
         assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)), APPLICATION_JSON, JSON_TABLE_BODY);
     }
 
+    public void testThatWeUseTheAcceptHeaderJsonForPaginatedTable() throws Exception {
+        assertResponse(
+            Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)),
+            APPLICATION_JSON,
+            PAGINATED_JSON_TABLE_BODY,
+            getPaginatedTable()
+        );
+    }
+
     public void testThatWeUseTheAcceptHeaderYaml() throws Exception {
         assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)), APPLICATION_YAML, YAML_TABLE_BODY);
     }
 
+    public void testThatWeUseTheAcceptHeaderYamlForPaginatedTable() throws Exception {
+        assertResponse(
+            Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)),
+            APPLICATION_YAML,
+            PAGINATED_YAML_TABLE_BODY,
+            getPaginatedTable()
+        );
+    }
+
     public void testThatWeUseTheAcceptHeaderSmile() throws Exception {
         assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_SMILE)), APPLICATION_SMILE);
     }
@@ -137,6 +159,15 @@ public void testThatWeUseTheAcceptHeaderText() throws Exception {
         assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)), TEXT_PLAIN, TEXT_TABLE_BODY);
     }
 
+    public void testThatWeUseTheAcceptHeaderTextForPaginatedTable() throws Exception {
+        assertResponse(
+            Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)),
+            TEXT_PLAIN,
+            PAGINATED_TEXT_TABLE_BODY,
+            getPaginatedTable()
+        );
+    }
+
     public void testIgnoreContentType() throws Exception {
         assertResponse(Collections.singletonMap(CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)), TEXT_PLAIN, TEXT_TABLE_BODY);
     }
@@ -261,6 +292,10 @@ public void testMultiSort() {
     }
 
     private RestResponse assertResponseContentType(Map<String, List<String>> headers, String mediaType) throws Exception {
+        return assertResponseContentType(headers, mediaType, table);
+    }
+
+    private RestResponse assertResponseContentType(Map<String, List<String>> headers, String mediaType, Table table) throws Exception {
         FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).build();
         table.startRow();
         table.addCell("foo");
@@ -282,7 +317,11 @@ public void sendResponse(RestResponse response) {}
     }
 
     private void assertResponse(Map<String, List<String>> headers, String mediaType, String body) throws Exception {
-        RestResponse response = assertResponseContentType(headers, mediaType);
+        assertResponse(headers, mediaType, body, table);
+    }
+
+    private void assertResponse(Map<String, List<String>> headers, String mediaType, String body, Table table) throws Exception {
+        RestResponse response = assertResponseContentType(headers, mediaType, table);
         assertThat(response.content().utf8ToString(), equalTo(body));
     }
 
@@ -294,4 +333,28 @@ private List<String> getHeaderNames(List<RestTable.DisplayHeader> headers) {
 
         return headerNames;
     }
+
+    private Table getPaginatedTable() {
+        PageToken pageToken = new PageToken("foo", "entities");
+        Table paginatedTable = new Table(pageToken);
+        addHeaders(paginatedTable);
+        return paginatedTable;
+    }
+
+    private void addHeaders(Table table) {
+        table.startHeaders();
+        table.addCell("bulk.foo", "alias:f;desc:foo");
+        table.addCell("bulk.bar", "alias:b;desc:bar");
+        // should be matched as well due to the aliases
+        table.addCell("aliasedBulk", "alias:bulkWhatever;desc:bar");
+        table.addCell("aliasedSecondBulk", "alias:foobar,bulkolicious,bulkotastic;desc:bar");
+        // no match
+        table.addCell("unmatched", "alias:un.matched;desc:bar");
+        // invalid alias
+        table.addCell("invalidAliasesBulk", "alias:,,,;desc:bar");
+        // timestamp
+        table.addCell("timestamp", "alias:ts");
+        table.addCell("epoch", "alias:t");
+        table.endHeaders();
+    }
 }
diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java
new file mode 100644
index 0000000000000..01464b489e26e
--- /dev/null
+++ b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java
@@ -0,0 +1,399 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rest.pagination;
+
+import org.opensearch.OpenSearchParseException;
+import org.opensearch.Version;
+import org.opensearch.cluster.ClusterName;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.routing.IndexRoutingTable;
+import org.opensearch.cluster.routing.RoutingTable;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE;
+import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE;
+import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE;
+import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
+
+public class IndexPaginationStrategyTests extends OpenSearchTestCase {
+
+    public void testRetrieveAllIndicesWithVaryingPageSize() {
+        List<Integer> indexNumberList = new ArrayList<>();
+        final int totalIndices = 100;
+        for (int indexNumber = 1; indexNumber <= 100; indexNumber++) {
+            indexNumberList.add(indexNumber);
+        }
+        // creating a cluster state with 100 indices
+        Collections.shuffle(indexNumberList, getRandom());
+        ClusterState clusterState = getRandomClusterState(indexNumberList);
+
+        // Checking pagination response for different pageSizes, which has a mix of even and odd numbers
+        // to ensure number of indices in last page is not always equal to pageSize.
+        List<Integer> pageSizeList = List.of(1, 6, 10, 13);
+        List<String> sortOrderList = List.of(PARAM_ASC_SORT_VALUE, PARAM_DESC_SORT_VALUE);
+        for (String sortOrder : sortOrderList) {
+            for (int pageSize : pageSizeList) {
+                String requestedToken = null;
+                int totalPagesToFetch = (int) Math.ceil(totalIndices / (pageSize * 1.0));
+                int indicesRemaining = totalIndices;
+                for (int pageNumber = 1; pageNumber <= totalPagesToFetch; pageNumber++) {
+                    PageParams pageParams = new PageParams(requestedToken, sortOrder, pageSize);
+                    IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+                    if (pageNumber < totalPagesToFetch) {
+                        assertNotNull(paginationStrategy.getResponseToken().getNextToken());
+                    } else {
+                        assertNull(paginationStrategy.getResponseToken().getNextToken());
+                    }
+                    requestedToken = paginationStrategy.getResponseToken().getNextToken();
+                    // Asserting all the indices received
+                    int responseItr = 0;
+                    if (PARAM_ASC_SORT_VALUE.equals(sortOrder)) {
+                        for (int indexNumber = (pageNumber - 1) * pageSize; indexNumber < Math.min(
+                            100,
+                            pageNumber * pageSize
+                        ); indexNumber++) {
+                            assertEquals("test-index-" + (indexNumber + 1), paginationStrategy.getRequestedEntities().get(responseItr));
+                            responseItr++;
+                        }
+                    } else {
+                        int endIndexNumberForPage = Math.max(indicesRemaining - pageSize, 0);
+                        for (; indicesRemaining > endIndexNumberForPage; indicesRemaining--) {
+                            assertEquals("test-index-" + indicesRemaining, paginationStrategy.getRequestedEntities().get(responseItr));
+                            responseItr++;
+                        }
+                    }
+                    assertEquals(responseItr, paginationStrategy.getRequestedEntities().size());
+                }
+            }
+        }
+    }
+
+    public void testRetrieveAllIndicesInAscOrderWhileIndicesGetCreatedAndDeleted() {
+        List<Integer> indexNumberList = new ArrayList<>();
+        List<Integer> deletedIndices = new ArrayList<>();
+        final int totalIndices = 100;
+        final int numIndicesToDelete = 10;
+        final int numIndicesToCreate = 5;
+        List<String> indicesFetched = new ArrayList<>();
+        for (int indexNumber = 1; indexNumber <= 100; indexNumber++) {
+            indexNumberList.add(indexNumber);
+        }
+        ClusterState clusterState = getRandomClusterState(indexNumberList);
+
+        int pageSize = 6;
+        String requestedToken = null;
+        int numPages = 0;
+        do {
+            numPages++;
+            PageParams pageParams = new PageParams(requestedToken, PARAM_ASC_SORT_VALUE, pageSize);
+            IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+            assertNotNull(paginationStrategy);
+            assertNotNull(paginationStrategy.getResponseToken());
+            requestedToken = paginationStrategy.getResponseToken().getNextToken();
+            // randomly deleting 10 indices after 3rd call
+            if (numPages == 3) {
+                deletedIndices = indexNumberList.subList(20, indexNumberList.size());
+                Collections.shuffle(deletedIndices, getRandom());
+                for (int pos = 0; pos < numIndicesToDelete; pos++) {
+                    clusterState = deleteIndexFromClusterState(clusterState, deletedIndices.get(pos));
+                }
+            }
+            // creating 5 indices after 5th call
+            if (numPages == 5) {
+                for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) {
+                    clusterState = addIndexToClusterState(clusterState, indexNumber);
+                }
+            }
+            if (requestedToken == null) {
+                assertEquals(paginationStrategy.getRequestedEntities().size(), 5);
+            } else {
+                assertEquals(paginationStrategy.getRequestedEntities().size(), pageSize);
+            }
+
+            indicesFetched.addAll(paginationStrategy.getRequestedEntities());
+        } while (Objects.nonNull(requestedToken));
+
+        assertEquals((int) Math.ceil((double) (totalIndices + numIndicesToCreate - numIndicesToDelete) / pageSize), numPages);
+        assertEquals(totalIndices + numIndicesToCreate - numIndicesToDelete, indicesFetched.size());
+
+        // none of the deleted index should appear in the list of fetched indices
+        for (int deletedIndexPos = 0; deletedIndexPos < numIndicesToDelete; deletedIndexPos++) {
+            assertFalse(indicesFetched.contains("test-index-" + deletedIndices.get(deletedIndexPos)));
+        }
+
+        // all the newly created indices should be present in the list of fetched indices
+        for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) {
+            assertTrue(indicesFetched.contains("test-index-" + indexNumber));
+        }
+    }
+
+    public void testRetrieveAllIndicesInDescOrderWhileIndicesGetCreatedAndDeleted() {
+        List<Integer> indexNumberList = new ArrayList<>();
+        List<Integer> deletedIndices = new ArrayList<>();
+        final int totalIndices = 100;
+        final int numIndicesToDelete = 9;
+        final int numIndicesToCreate = 5;
+        List<String> indicesFetched = new ArrayList<>();
+        for (int indexNumber = 1; indexNumber <= 100; indexNumber++) {
+            indexNumberList.add(indexNumber);
+        }
+        ClusterState clusterState = getRandomClusterState(indexNumberList);
+
+        int pageSize = 6;
+        String requestedToken = null;
+        int numPages = 0;
+        do {
+            numPages++;
+            PageParams pageParams = new PageParams(requestedToken, PARAM_DESC_SORT_VALUE, pageSize);
+            IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+            assertNotNull(paginationStrategy);
+            assertNotNull(paginationStrategy.getResponseToken());
+            requestedToken = paginationStrategy.getResponseToken().getNextToken();
+            // randomly deleting 10 indices after 3rd call
+            if (numPages == 3) {
+                deletedIndices = indexNumberList.subList(0, 80);
+                Collections.shuffle(deletedIndices, getRandom());
+                for (int pos = 0; pos < numIndicesToDelete; pos++) {
+                    clusterState = deleteIndexFromClusterState(clusterState, deletedIndices.get(pos));
+                }
+            }
+            // creating 5 indices after 5th call
+            if (numPages == 5) {
+                for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) {
+                    clusterState = addIndexToClusterState(clusterState, indexNumber);
+                }
+            }
+            if (requestedToken == null) {
+                assertEquals(paginationStrategy.getRequestedEntities().size(), (totalIndices - numIndicesToDelete) % pageSize);
+            } else {
+                assertEquals(paginationStrategy.getRequestedEntities().size(), pageSize);
+            }
+
+            indicesFetched.addAll(paginationStrategy.getRequestedEntities());
+        } while (Objects.nonNull(requestedToken));
+
+        assertEquals((int) Math.ceil((double) (totalIndices - numIndicesToDelete) / pageSize), numPages);
+        assertEquals(totalIndices - numIndicesToDelete, indicesFetched.size());
+
+        // none of the deleted index should appear in the list of fetched indices
+        for (int deletedIndexPos = 0; deletedIndexPos < numIndicesToDelete; deletedIndexPos++) {
+            assertFalse(indicesFetched.contains("test-index-" + deletedIndices.get(deletedIndexPos)));
+        }
+
+        // none of the newly created indices should be present in the list of fetched indices
+        for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) {
+            assertFalse(indicesFetched.contains("test-index-" + indexNumber));
+        }
+    }
+
+    public void testRetrieveIndicesWithSizeOneAndCurrentIndexGetsDeletedAscOrder() {
+        // Query1 with 4 indices in clusterState (test-index1,2,3,4)
+        ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4));
+        PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 1);
+        IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertPaginationResult(paginationStrategy, 1, true);
+        assertEquals("test-index-1", paginationStrategy.getRequestedEntities().get(0));
+
+        // Adding index5 to clusterState, before executing next query.
+        clusterState = addIndexToClusterState(clusterState, 5);
+        pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1);
+        paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertPaginationResult(paginationStrategy, 1, true);
+        assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0));
+
+        // Deleting test-index-2 which has already been displayed, still test-index-3 should get displayed
+        clusterState = deleteIndexFromClusterState(clusterState, 2);
+        pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1);
+        paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertPaginationResult(paginationStrategy, 1, true);
+        assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0));
+
+        // Deleting test-index-4 which is not yet displayed which otherwise should have been displayed in the following query
+        // instead test-index-5 should now get displayed.
+        clusterState = deleteIndexFromClusterState(clusterState, 4);
+        pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1);
+        paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertPaginationResult(paginationStrategy, 1, false);
+        assertEquals("test-index-5", paginationStrategy.getRequestedEntities().get(0));
+
+    }
+
+    public void testRetrieveIndicesWithSizeOneAndCurrentIndexGetsDeletedDescOrder() {
+        // Query1 with 4 indices in clusterState (test-index1,2,3,4).
+        ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4));
+        PageParams pageParams = new PageParams(null, PARAM_DESC_SORT_VALUE, 1);
+        IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertPaginationResult(paginationStrategy, 1, true);
+        assertEquals("test-index-4", paginationStrategy.getRequestedEntities().get(0));
+
+        // adding test-index-5 to clusterState, before executing next query.
+        clusterState = addIndexToClusterState(clusterState, 5);
+        pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1);
+        paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertPaginationResult(paginationStrategy, 1, true);
+        assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0));
+
+        // Deleting test-index-3 which has already been displayed, still index2 should get displayed.
+        clusterState = deleteIndexFromClusterState(clusterState, 3);
+        pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1);
+        paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertPaginationResult(paginationStrategy, 1, true);
+        assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0));
+
+        // Deleting test-index-1 which is not yet displayed which otherwise should have been displayed in the following query.
+        clusterState = deleteIndexFromClusterState(clusterState, 1);
+        pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1);
+        paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertPaginationResult(paginationStrategy, 0, false);
+    }
+
+    public void testRetrieveIndicesWithMultipleDeletionsAtOnceAscOrder() {
+        // Query1 with 5 indices in clusterState (test-index1,2,3,4,5).
+        ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5));
+        PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 1);
+        IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertEquals(1, paginationStrategy.getRequestedEntities().size());
+        assertEquals("test-index-1", paginationStrategy.getRequestedEntities().get(0));
+        assertNotNull(paginationStrategy.getResponseToken().getNextToken());
+
+        // executing next query without any changes to clusterState
+        pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1);
+        paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertEquals(1, paginationStrategy.getRequestedEntities().size());
+        assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0));
+        assertNotNull(paginationStrategy.getResponseToken().getNextToken());
+
+        // Deleting test-index-1, test-index-2 & test-index-3 and executing next query. test-index-4 should get displayed.
+        clusterState = deleteIndexFromClusterState(clusterState, 1);
+        clusterState = deleteIndexFromClusterState(clusterState, 2);
+        clusterState = deleteIndexFromClusterState(clusterState, 3);
+        pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1);
+        paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertEquals(1, paginationStrategy.getRequestedEntities().size());
+        assertEquals("test-index-4", paginationStrategy.getRequestedEntities().get(0));
+        assertNotNull(paginationStrategy.getResponseToken().getNextToken());
+
+        // Executing the last query without any further change. Should result in test-index-5 and nextToken as null.
+        pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1);
+        paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertEquals(1, paginationStrategy.getRequestedEntities().size());
+        assertEquals("test-index-5", paginationStrategy.getRequestedEntities().get(0));
+        assertNull(paginationStrategy.getResponseToken().getNextToken());
+    }
+
+    public void testRetrieveIndicesWithTokenModifiedToQueryBeyondTotal() {
+        ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10));
+        PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 10);
+        IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertEquals(10, paginationStrategy.getRequestedEntities().size());
+        assertNull(paginationStrategy.getResponseToken().getNextToken());
+        // creating a token with last sent index as test-index-10
+        String token = clusterState.metadata().indices().get("test-index-10").getCreationDate() + "|" + "test-index-10";
+        pageParams = new PageParams(Base64.getEncoder().encodeToString(token.getBytes(UTF_8)), PARAM_ASC_SORT_VALUE, 10);
+        paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState);
+        assertEquals(0, paginationStrategy.getRequestedEntities().size());
+        assertNull(paginationStrategy.getResponseToken().getNextToken());
+    }
+
+    public void testCreatingIndexStrategyPageTokenWithRequestedTokenNull() {
+        try {
+            new IndexPaginationStrategy.IndexStrategyToken(null);
+            fail("expected exception");
+        } catch (Exception e) {
+            assert e.getMessage().contains("requestedTokenString can not be null");
+        }
+    }
+
+    public void testIndexStrategyPageTokenWithWronglyEncryptedRequestToken() {
+        assertThrows(OpenSearchParseException.class, () -> new IndexPaginationStrategy.IndexStrategyToken("3%4%5"));
+    }
+
+    public void testIndexStrategyPageTokenWithIncorrectNumberOfElementsInRequestedToken() {
+        assertThrows(
+            OpenSearchParseException.class,
+            () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1725361543"))
+        );
+        assertThrows(
+            OpenSearchParseException.class,
+            () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1|1725361543|index|12345"))
+        );
+    }
+
+    public void testIndexStrategyPageTokenWithInvalidValuesInRequestedToken() {
+        assertThrows(
+            OpenSearchParseException.class,
+            () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("-1725361543|index"))
+        );
+    }
+
+    public void testCreatingIndexStrategyPageTokenWithNameOfLastRespondedIndexNull() {
+        try {
+            new IndexPaginationStrategy.IndexStrategyToken(1234l, null);
+            fail("expected exception");
+        } catch (Exception e) {
+            assert e.getMessage().contains("index name should be provided");
+        }
+    }
+
+    /**
+     * @param indexNumbers would be used to create indices having names with integer appended after foo, like foo1, foo2.
+     * @return random clusterState consisting of indices having their creation times set to the integer used to name them.
+     */
+    private ClusterState getRandomClusterState(List<Integer> indexNumbers) {
+        ClusterState clusterState = ClusterState.builder(new ClusterName("test"))
+            .metadata(Metadata.builder().build())
+            .routingTable(RoutingTable.builder().build())
+            .build();
+        for (Integer indexNumber : indexNumbers) {
+            clusterState = addIndexToClusterState(clusterState, indexNumber);
+        }
+        return clusterState;
+    }
+
+    private ClusterState addIndexToClusterState(ClusterState clusterState, int indexNumber) {
+        IndexMetadata indexMetadata = IndexMetadata.builder("test-index-" + indexNumber)
+            .settings(
+                settings(Version.CURRENT).put(SETTING_CREATION_DATE, Instant.now().plus(indexNumber, ChronoUnit.SECONDS).toEpochMilli())
+            )
+            .numberOfShards(between(1, 10))
+            .numberOfReplicas(randomInt(20))
+            .build();
+        IndexRoutingTable.Builder indexRoutingTableBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex());
+        return ClusterState.builder(clusterState)
+            .metadata(Metadata.builder(clusterState.metadata()).put(indexMetadata, true).build())
+            .routingTable(RoutingTable.builder(clusterState.routingTable()).add(indexRoutingTableBuilder).build())
+            .build();
+    }
+
+    private ClusterState deleteIndexFromClusterState(ClusterState clusterState, int indexNumber) {
+        return ClusterState.builder(clusterState)
+            .metadata(Metadata.builder(clusterState.metadata()).remove("test-index-" + indexNumber))
+            .routingTable(RoutingTable.builder(clusterState.routingTable()).remove("test-index-" + indexNumber).build())
+            .build();
+    }
+
+    private void assertPaginationResult(IndexPaginationStrategy paginationStrategy, int expectedEntities, boolean tokenExpected) {
+        assertNotNull(paginationStrategy);
+        assertEquals(expectedEntities, paginationStrategy.getRequestedEntities().size());
+        assertNotNull(paginationStrategy.getResponseToken());
+        assertEquals(tokenExpected, Objects.nonNull(paginationStrategy.getResponseToken().getNextToken()));
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java
index 4362ce48003cc..14532e30f8984 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java
@@ -128,6 +128,7 @@ public void testNumericKeys() throws IOException {
         );
         assertThat(builder.getName(), equalTo("test"));
         assertThat(builder.field(), equalTo("f"));
+        assertThat(builder.fieldName(), equalTo("f"));
         assertThat(builder.ranges, equalTo(List.of(new RangeAggregator.Range("1", null, 0d))));
     }
 }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java
index a02ca3cf877ad..683ada76c7683 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java
@@ -37,4 +37,32 @@ public void testSearchBackpressureSettingValidateInvalidMode() {
             () -> new SearchBackpressureSettings(settings, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
         );
     }
+
+    public void testInvalidCancellationRate() {
+        Settings settings1 = Settings.builder().put("search_backpressure.search_task.cancellation_rate", randomFrom(-1, 0)).build();
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> new SearchBackpressureSettings(settings1, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+
+        Settings settings2 = Settings.builder().put("search_backpressure.search_shard_task.cancellation_rate", randomFrom(-1, 0)).build();
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> new SearchBackpressureSettings(settings2, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+    }
+
+    public void testInvalidCancellationRatio() {
+        Settings settings1 = Settings.builder().put("search_backpressure.search_task.cancellation_ratio", randomFrom(-1, 0)).build();
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> new SearchBackpressureSettings(settings1, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+
+        Settings settings2 = Settings.builder().put("search_backpressure.search_shard_task.cancellation_ratio", randomFrom(-1, 0)).build();
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> new SearchBackpressureSettings(settings2, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+    }
 }
diff --git a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
index 9697f4cee0d58..da8ccc9e121e0 100644
--- a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
+++ b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
@@ -421,6 +421,27 @@ public void testDerivedFieldsParsingAndSerializationObjectType() throws IOExcept
         }
     }
 
+    public void testSearchPipelineParsingAndSerialization() throws IOException {
+        String restContent = "{ \"query\": { \"match_all\": {} }, \"from\": 0, \"size\": 10, \"search_pipeline\": \"my_pipeline\" }";
+        String expectedContent = "{\"from\":0,\"size\":10,\"query\":{\"match_all\":{\"boost\":1.0}},\"search_pipeline\":\"my_pipeline\"}";
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+            SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser);
+            searchSourceBuilder = rewrite(searchSourceBuilder);
+
+            try (BytesStreamOutput output = new BytesStreamOutput()) {
+                searchSourceBuilder.writeTo(output);
+                try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                    SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in);
+                    String actualContent = deserializedBuilder.toString();
+                    assertEquals(expectedContent, actualContent);
+                    assertEquals(searchSourceBuilder.hashCode(), deserializedBuilder.hashCode());
+                    assertNotSame(searchSourceBuilder, deserializedBuilder);
+                }
+            }
+        }
+    }
+
     public void testAggsParsing() throws IOException {
         {
             String restContent = "{\n"
diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java
index f5857922fdff2..b52205996f34b 100644
--- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java
@@ -969,6 +969,64 @@ public void testInlinePipeline() throws Exception {
         }
     }
 
+    public void testInlineDefinedPipeline() throws Exception {
+        SearchPipelineService searchPipelineService = createWithProcessors();
+
+        SearchPipelineMetadata metadata = new SearchPipelineMetadata(
+            Map.of(
+                "p1",
+                new PipelineConfiguration(
+                    "p1",
+                    new BytesArray(
+                        "{"
+                            + "\"request_processors\": [{ \"scale_request_size\": { \"scale\" : 2 } }],"
+                            + "\"response_processors\": [{ \"fixed_score\": { \"score\" : 2 } }]"
+                            + "}"
+                    ),
+                    MediaTypeRegistry.JSON
+                )
+            )
+
+        );
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
+        ClusterState previousState = clusterState;
+        clusterState = ClusterState.builder(clusterState)
+            .metadata(Metadata.builder().putCustom(SearchPipelineMetadata.TYPE, metadata))
+            .build();
+        searchPipelineService.applyClusterState(new ClusterChangedEvent("", clusterState, previousState));
+
+        SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().size(100).pipeline("p1");
+        SearchRequest searchRequest = new SearchRequest().source(sourceBuilder);
+        searchRequest.pipeline(searchRequest.source().pipeline());
+
+        // Verify pipeline
+        PipelinedRequest pipelinedRequest = syncTransformRequest(
+            searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver)
+        );
+        Pipeline pipeline = pipelinedRequest.getPipeline();
+        assertEquals("p1", pipeline.getId());
+        assertEquals(1, pipeline.getSearchRequestProcessors().size());
+        assertEquals(1, pipeline.getSearchResponseProcessors().size());
+
+        // Verify that pipeline transforms request
+        assertEquals(200, pipelinedRequest.source().size());
+
+        int size = 10;
+        SearchHit[] hits = new SearchHit[size];
+        for (int i = 0; i < size; i++) {
+            hits[i] = new SearchHit(i, "doc" + i, Collections.emptyMap(), Collections.emptyMap());
+            hits[i].score(i);
+        }
+        SearchHits searchHits = new SearchHits(hits, new TotalHits(size * 2, TotalHits.Relation.EQUAL_TO), size);
+        SearchResponseSections searchResponseSections = new SearchResponseSections(searchHits, null, null, false, false, null, 0);
+        SearchResponse searchResponse = new SearchResponse(searchResponseSections, null, 1, 1, 0, 10, null, null);
+
+        SearchResponse transformedResponse = syncTransformResponse(pipelinedRequest, searchResponse);
+        for (int i = 0; i < size; i++) {
+            assertEquals(2.0, transformedResponse.getHits().getHits()[i].getScore(), 0.0001);
+        }
+    }
+
     public void testInfo() {
         SearchPipelineService searchPipelineService = createWithProcessors();
         SearchPipelineInfo info = searchPipelineService.info();
diff --git a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java
index 9b8cd1b5f1ce0..ced952db555aa 100644
--- a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java
+++ b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java
@@ -196,6 +196,7 @@ protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField
         assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse());
         if (expectedType == SortField.Type.CUSTOM) {
             assertEquals(builder.getFieldName(), sortField.getField());
+            assertEquals(builder.fieldName(), sortField.getField());
         }
         assertEquals(DocValueFormat.RAW, format);
     }
diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java
index 95a8267734a07..72798f7691eb1 100644
--- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java
@@ -39,6 +39,7 @@
 import org.opensearch.common.blobstore.BlobMetadata;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.blobstore.BlobStore;
+import org.opensearch.common.blobstore.DeleteResult;
 import org.opensearch.common.blobstore.fs.FsBlobContainer;
 import org.opensearch.common.blobstore.fs.FsBlobStore;
 import org.opensearch.common.blobstore.stream.read.ReadContext;
@@ -63,6 +64,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 
@@ -335,5 +337,15 @@ public boolean remoteIntegrityCheckSupported() {
         public BlobContainer getDelegate() {
             return delegate;
         }
+
+        @Override
+        public void deleteAsync(ActionListener<DeleteResult> completionListener) {
+            throw new RuntimeException("deleteAsync not supported");
+        }
+
+        @Override
+        public void deleteBlobsAsyncIgnoringIfNotExists(List<String> blobNames, ActionListener<Void> completionListener) {
+            throw new RuntimeException("deleteBlobsAsyncIgnoringIfNotExists not supported");
+        }
     }
 }
diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
index 350c6f9ae8f6b..d17e661615b0d 100644
--- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
@@ -1923,11 +1923,6 @@ private final class TestClusterNode {
                         protected PrioritizedOpenSearchThreadPoolExecutor createThreadPoolExecutor() {
                             return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue, threadPool);
                         }
-
-                        @Override
-                        protected void connectToNodesAndWait(ClusterState newClusterState) {
-                            // don't do anything, and don't block
-                        }
                     }
                 );
                 recoverySettings = new RecoverySettings(settings, clusterSettings);
@@ -2094,7 +2089,7 @@ public void onFailure(final Exception e) {
                     rerouteService,
                     threadPool
                 );
-                nodeConnectionsService = new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService);
+                nodeConnectionsService = createTestNodeConnectionsService(clusterService.getSettings(), threadPool, transportService);
                 final MetadataMappingService metadataMappingService = new MetadataMappingService(clusterService, indicesService);
                 indicesClusterStateService = new IndicesClusterStateService(
                     settings,
@@ -2434,7 +2429,8 @@ public void onFailure(final Exception e) {
                             nodeEnv,
                             indicesService,
                             namedXContentRegistry
-                        )
+                        ),
+                        new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
                     )
                 );
                 actions.put(
@@ -2492,6 +2488,24 @@ protected void assertSnapshotOrGenericThread() {
                 }
             }
 
+            public NodeConnectionsService createTestNodeConnectionsService(
+                Settings settings,
+                ThreadPool threadPool,
+                TransportService transportService
+            ) {
+                return new NodeConnectionsService(settings, threadPool, transportService) {
+                    @Override
+                    public void connectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletion) {
+                        // just update targetsByNode to ensure disconnect runs for these nodes
+                        // we rely on disconnect to run for keeping track of pendingDisconnects and ensuring node-joins can happen
+                        for (final DiscoveryNode discoveryNode : discoveryNodes) {
+                            this.targetsByNode.put(discoveryNode, createConnectionTarget(discoveryNode));
+                        }
+                        onCompletion.run();
+                    }
+                };
+            }
+
             public ClusterInfoService getMockClusterInfoService() {
                 return clusterInfoService;
             }
@@ -2563,10 +2577,11 @@ public void start(ClusterState initialState) {
                     new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE),
                     null
                 );
+                coordinator.setNodeConnectionsService(nodeConnectionsService);
                 clusterManagerService.setClusterStatePublisher(coordinator);
-                coordinator.start();
                 clusterService.getClusterApplierService().setNodeConnectionsService(nodeConnectionsService);
                 nodeConnectionsService.start();
+                coordinator.start();
                 clusterService.start();
                 indicesService.start();
                 indicesClusterStateService.start();
diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java
index 8fd1f44286094..d79cb62b6b7ac 100644
--- a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java
@@ -214,7 +214,11 @@ public void testSerDeRemoteStoreIndexShallowCopy() throws IOException {
                     assert (curr_entry.remoteStoreIndexShallowCopy() == false);
                 }
             }
-            try (StreamInput in = out.bytes().streamInput()) {
+
+            BytesStreamOutput out2 = new BytesStreamOutput();
+            out2.setVersion(Version.V_2_9_0);
+            snapshotsInProgress.writeTo(out2);
+            try (StreamInput in = out2.bytes().streamInput()) {
                 in.setVersion(Version.V_2_9_0);
                 actualSnapshotsInProgress = new SnapshotsInProgress(in);
                 assert in.available() == 0;
diff --git a/server/src/test/java/org/opensearch/test/NoopDiscovery.java b/server/src/test/java/org/opensearch/test/NoopDiscovery.java
index 42d3f1887ab4d..c35503a556db6 100644
--- a/server/src/test/java/org/opensearch/test/NoopDiscovery.java
+++ b/server/src/test/java/org/opensearch/test/NoopDiscovery.java
@@ -32,6 +32,7 @@
 package org.opensearch.test;
 
 import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.NodeConnectionsService;
 import org.opensearch.common.lifecycle.Lifecycle;
 import org.opensearch.common.lifecycle.LifecycleListener;
 import org.opensearch.core.action.ActionListener;
@@ -55,6 +56,11 @@ public void startInitialJoin() {
 
     }
 
+    @Override
+    public void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) {
+
+    }
+
     @Override
     public Lifecycle.State lifecycleState() {
         return null;
diff --git a/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java b/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java
index 1d734a56ef189..fdf762aa096f0 100644
--- a/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java
+++ b/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java
@@ -320,6 +320,50 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
         assertEquals(0, nodeDisconnectedCount.get());
     }
 
+    public void testConnectFailsWhenDisconnectIsPending() {
+        AtomicInteger nodeConnectedCount = new AtomicInteger();
+        AtomicInteger nodeDisconnectedCount = new AtomicInteger();
+        connectionManager.addListener(new TransportConnectionListener() {
+            @Override
+            public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) {
+                nodeConnectedCount.incrementAndGet();
+            }
+
+            @Override
+            public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) {
+                nodeDisconnectedCount.incrementAndGet();
+            }
+        });
+
+        DiscoveryNode node = new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT);
+        ConnectionManager.ConnectionValidator validator = (c, p, l) -> l.onResponse(null);
+        Transport.Connection connection = new TestConnect(node);
+        doAnswer(invocationOnMock -> {
+            ActionListener<Transport.Connection> listener = (ActionListener<Transport.Connection>) invocationOnMock.getArguments()[2];
+            listener.onResponse(connection);
+            return null;
+        }).when(transport).openConnection(eq(node), eq(connectionProfile), any(ActionListener.class));
+        assertFalse(connectionManager.nodeConnected(node));
+
+        // Mark connection as pending disconnect, any connection attempt should fail
+        connectionManager.setPendingDisconnection(node);
+        PlainActionFuture<Void> fut = new PlainActionFuture<>();
+        connectionManager.connectToNode(node, connectionProfile, validator, fut);
+        expectThrows(IllegalStateException.class, () -> fut.actionGet());
+
+        // clear the pending disconnect and assert that connection succeeds
+        connectionManager.clearPendingDisconnections();
+        assertFalse(connectionManager.nodeConnected(node));
+        PlainActionFuture.get(
+            future -> connectionManager.connectToNode(node, connectionProfile, validator, ActionListener.map(future, x -> null))
+        );
+        assertFalse(connection.isClosed());
+        assertTrue(connectionManager.nodeConnected(node));
+        assertEquals(1, connectionManager.size());
+        assertEquals(1, nodeConnectedCount.get());
+        assertEquals(0, nodeDisconnectedCount.get());
+    }
+
     private static class TestConnect extends CloseableConnection {
 
         private final DiscoveryNode node;
diff --git a/settings.gradle b/settings.gradle
index b79c2aee135fc..8412d198a2a29 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -10,7 +10,7 @@
  */
 
 plugins {
-  id "com.gradle.develocity" version "3.18"
+  id "com.gradle.develocity" version "3.18.1"
 }
 
 ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE')
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index b5cd12ef0c11f..bc63362980bea 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -55,7 +55,7 @@ dependencies {
     exclude group: 'com.nimbusds'
     exclude module: "commons-configuration2"
   }
-  api "dnsjava:dnsjava:3.6.1"
+  api "dnsjava:dnsjava:3.6.2"
   api "org.codehaus.jettison:jettison:${versions.jettison}"
   api "org.apache.commons:commons-compress:${versions.commonscompress}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
@@ -74,12 +74,12 @@ dependencies {
   api 'org.apache.zookeeper:zookeeper:3.9.2'
   api "org.apache.commons:commons-text:1.12.0"
   api "commons-net:commons-net:3.11.1"
-  api "ch.qos.logback:logback-core:1.5.6"
+  api "ch.qos.logback:logback-core:1.5.8"
   api "ch.qos.logback:logback-classic:1.2.13"
   api "org.jboss.xnio:xnio-nio:3.8.16.Final"
-  api 'org.jline:jline:3.26.3'
+  api 'org.jline:jline:3.27.0'
   api 'org.apache.commons:commons-configuration2:2.11.0'
-  api 'com.nimbusds:nimbus-jose-jwt:9.40'
+  api 'com.nimbusds:nimbus-jose-jwt:9.41.1'
   api ('org.apache.kerby:kerb-admin:2.1.0') {
     exclude group: "org.jboss.xnio"
     exclude group: "org.jline"
diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
index 35ca5d80aeb4e..ded457601c0ae 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
@@ -125,7 +125,8 @@ List<NodeStats> adjustNodesStats(List<NodeStats> nodesStats) {
                 nodeStats.getSegmentReplicationRejectionStats(),
                 nodeStats.getRepositoriesStats(),
                 nodeStats.getAdmissionControlStats(),
-                nodeStats.getNodeCacheStats()
+                nodeStats.getNodeCacheStats(),
+                nodeStats.getRemoteStoreNodeStats()
             );
         }).collect(Collectors.toList());
     }
diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
index b432e5411404e..3efcc538a1b25 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
@@ -55,6 +55,7 @@
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.allocation.AllocationService;
 import org.opensearch.cluster.service.ClusterApplierService;
 import org.opensearch.cluster.service.ClusterService;
@@ -1150,9 +1151,12 @@ protected Optional<DisruptableMockTransport> getDisruptableMockTransport(Transpo
                     new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
                 );
                 clusterService = new ClusterService(settings, clusterSettings, clusterManagerService, clusterApplierService);
-                clusterService.setNodeConnectionsService(
-                    new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService)
+                NodeConnectionsService nodeConnectionsService = createTestNodeConnectionsService(
+                    clusterService.getSettings(),
+                    threadPool,
+                    transportService
                 );
+                clusterService.setNodeConnectionsService(nodeConnectionsService);
                 repositoriesService = new RepositoriesService(
                     settings,
                     clusterService,
@@ -1187,6 +1191,7 @@ protected Optional<DisruptableMockTransport> getDisruptableMockTransport(Transpo
                     new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE),
                     null
                 );
+                coordinator.setNodeConnectionsService(nodeConnectionsService);
                 clusterManagerService.setClusterStatePublisher(coordinator);
                 final GatewayService gatewayService = new GatewayService(
                     settings,
@@ -1588,6 +1593,24 @@ public void onNodeAck(DiscoveryNode node, Exception e) {
         }
     }
 
+    public static NodeConnectionsService createTestNodeConnectionsService(
+        Settings settings,
+        ThreadPool threadPool,
+        TransportService transportService
+    ) {
+        return new NodeConnectionsService(settings, threadPool, transportService) {
+            @Override
+            public void connectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletion) {
+                // just update targetsByNode to ensure disconnect runs for these nodes
+                // we rely on disconnect to run for keeping track of pendingDisconnects and ensuring node-joins can happen
+                for (final DiscoveryNode discoveryNode : discoveryNodes) {
+                    this.targetsByNode.put(discoveryNode, createConnectionTarget(discoveryNode));
+                }
+                onCompletion.run();
+            }
+        };
+    }
+
     static class DisruptableClusterApplierService extends ClusterApplierService {
         private final String nodeName;
         private final DeterministicTaskQueue deterministicTaskQueue;
@@ -1641,11 +1664,6 @@ public void onNewClusterState(String source, Supplier<ClusterState> clusterState
             }
         }
 
-        @Override
-        protected void connectToNodesAndWait(ClusterState newClusterState) {
-            // don't do anything, and don't block
-        }
-
         @Override
         protected boolean applicationMayFail() {
             return this.applicationMayFail;
diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java
index 1ad18bf89d5ba..4f692c7bc8f62 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java
@@ -92,6 +92,12 @@ public void runAllRunnableTasks() {
         }
     }
 
+    public void runAllRunnableTasksInEnqueuedOrder() {
+        while (hasRunnableTasks()) {
+            runTask(0);
+        }
+    }
+
     public void runAllTasks() {
         while (hasDeferredTasks() || hasRunnableTasks()) {
             if (hasDeferredTasks() && random.nextBoolean()) {
@@ -141,6 +147,11 @@ public void runRandomTask() {
         runTask(RandomNumbers.randomIntBetween(random, 0, runnableTasks.size() - 1));
     }
 
+    public void runNextTask() {
+        assert hasRunnableTasks();
+        runTask(0);
+    }
+
     private void runTask(final int index) {
         final Runnable task = runnableTasks.remove(index);
         logger.trace("running task {} of {}: {}", index, runnableTasks.size() + 1, task);
diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java
index a8bb10fe20752..0bfa70a771f65 100644
--- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -665,6 +665,16 @@ public void onTimeout(TimeValue timeout) {
         }
     }
 
+    protected ActionFuture<AcknowledgedResponse> deleteSnapshotBlockedOnClusterManager(String repoName, String snapshotName) {
+        blockClusterManagerFromDeletingIndexNFile(repoName);
+        return deleteSnapshot(repoName, snapshotName);
+    }
+
+    protected ActionFuture<AcknowledgedResponse> deleteSnapshot(String repoName, String snapshotName) {
+        logger.info("--> Deleting snapshot [{}] to repo [{}]", snapshotName, repoName);
+        return clusterAdmin().prepareDeleteSnapshot(repoName, snapshotName).execute();
+    }
+
     protected ActionFuture<CreateSnapshotResponse> startFullSnapshotBlockedOnDataNode(String snapshotName, String repoName, String dataNode)
         throws InterruptedException {
         blockDataNode(repoName, dataNode);
diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
index 7adff82e72245..fa5fb736f518f 100644
--- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
@@ -2752,6 +2752,7 @@ public void ensureEstimatedStats() {
                     false,
                     false,
                     false,
+                    false,
                     false
                 );
                 assertThat(
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index e474ef202b235..e27ff311c06f6 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -403,7 +403,7 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase {
 
     private static Boolean segmentsPathFixedPrefix;
 
-    private static Boolean snapshotShardPathFixedPrefix;
+    protected static Boolean snapshotShardPathFixedPrefix;
 
     private Path remoteStoreRepositoryPath;
 
@@ -2795,6 +2795,26 @@ public static Settings buildRemoteStoreNodeAttributes(
         );
     }
 
+    public static Settings buildRemoteStateNodeAttributes(String stateRepoName, Path stateRepoPath, String stateRepoType) {
+        String stateRepoTypeAttributeKey = String.format(
+            Locale.getDefault(),
+            "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
+            stateRepoName
+        );
+        String stateRepoSettingsAttributeKeyPrefix = String.format(
+            Locale.getDefault(),
+            "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
+            stateRepoName
+        );
+        String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey();
+        Settings.Builder settings = Settings.builder()
+            .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, stateRepoName)
+            .put(stateRepoTypeAttributeKey, stateRepoType)
+            .put(stateRepoSettingsAttributeKeyPrefix + "location", stateRepoPath)
+            .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable);
+        return settings.build();
+    }
+
     private static Settings buildRemoteStoreNodeAttributes(
         String segmentRepoName,
         Path segmentRepoPath,
@@ -2850,16 +2870,6 @@ private static Settings buildRemoteStoreNodeAttributes(
             "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
             translogRepoName
         );
-        String stateRepoTypeAttributeKey = String.format(
-            Locale.getDefault(),
-            "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
-            segmentRepoName
-        );
-        String stateRepoSettingsAttributeKeyPrefix = String.format(
-            Locale.getDefault(),
-            "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
-            segmentRepoName
-        );
         String routingTableRepoAttributeKey = null, routingTableRepoSettingsAttributeKeyPrefix = null;
         if (routingTableRepoName != null) {
             routingTableRepoAttributeKey = String.format(
@@ -2885,10 +2895,7 @@ private static Settings buildRemoteStoreNodeAttributes(
             .put(translogRepoTypeAttributeKey, translogRepoType)
             .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath)
             .put(translogRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable)
-            .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)
-            .put(stateRepoTypeAttributeKey, segmentRepoType)
-            .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
-            .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable);
+            .put(buildRemoteStateNodeAttributes(segmentRepoName, segmentRepoPath, segmentRepoType));
         if (routingTableRepoName != null) {
             settings.put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName)
                 .put(routingTableRepoAttributeKey, routingTableRepoType)
@@ -2904,7 +2911,7 @@ private static Settings buildRemoteStoreNodeAttributes(
         settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean());
         settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : "");
         settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : "");
-        settings.put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), segmentsPathFixedPrefix ? "c" : "");
+        settings.put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), snapshotShardPathFixedPrefix ? "c" : "");
         return settings.build();
     }
 
diff --git a/test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java b/test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java
new file mode 100644
index 0000000000000..030f399a5bcc0
--- /dev/null
+++ b/test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java
@@ -0,0 +1,74 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.test;
+
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Property;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test logs appender that provides functionality to extract specific logs/exception messages and wait for it to show up
+ * @opensearch.internal
+ */
+public class TestLogsAppender extends AbstractAppender {
+    private final List<String> capturedLogs = new ArrayList<>();
+    private final List<String> messagesToCapture;
+
+    public TestLogsAppender(List<String> messagesToCapture) {
+        super("TestAppender", null, PatternLayout.createDefaultLayout(), false, Property.EMPTY_ARRAY);
+        this.messagesToCapture = messagesToCapture;
+        start();
+    }
+
+    @Override
+    public void append(LogEvent event) {
+        if (shouldCaptureMessage(event.getMessage().getFormattedMessage())) capturedLogs.add(event.getMessage().getFormattedMessage());
+        if (event.getThrown() != null) {
+            if (shouldCaptureMessage(event.getThrown().toString())) capturedLogs.add(event.getThrown().toString());
+            for (StackTraceElement element : event.getThrown().getStackTrace())
+                if (shouldCaptureMessage(element.toString())) capturedLogs.add(element.toString());
+        }
+    }
+
+    public boolean shouldCaptureMessage(String log) {
+        return messagesToCapture.stream().anyMatch(log::contains);
+    }
+
+    public List<String> getCapturedLogs() {
+        return new ArrayList<>(capturedLogs);
+    }
+
+    public boolean waitForLog(String expectedLog, long timeout, TimeUnit unit) {
+        long startTime = System.currentTimeMillis();
+        long timeoutInMillis = unit.toMillis(timeout);
+
+        while (System.currentTimeMillis() - startTime < timeoutInMillis) {
+            if (capturedLogs.stream().anyMatch(log -> log.contains(expectedLog))) {
+                return true;
+            }
+            try {
+                Thread.sleep(100); // Wait for 100ms before checking again
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        }
+
+        return false;
+    }
+
+    // Clear captured logs
+    public void clearCapturedLogs() {
+        capturedLogs.clear();
+    }
+}
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java
index c7a0fe35b0237..fec1699c9ef64 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java
@@ -40,25 +40,20 @@
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
-import org.opensearch.identity.IdentityService;
 import org.opensearch.rest.RestController;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.tasks.Task;
 import org.opensearch.tasks.TaskListener;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.client.NoOpNodeClient;
-import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.usage.UsageService;
 import org.junit.After;
 import org.junit.Before;
 
 import java.util.Collections;
-import java.util.List;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiFunction;
 
-import static org.mockito.Mockito.mock;
-
 /**
  * A common base class for Rest*ActionTests. Provides access to a {@link RestController}
  * that can be used to register individual REST actions, and test request handling.
@@ -70,15 +65,7 @@ public abstract class RestActionTestCase extends OpenSearchTestCase {
     @Before
     public void setUpController() {
         verifyingClient = new VerifyingClient(this.getTestName());
-        final IdentityService identityService = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), List.of());
-        controller = new RestController(
-            Collections.emptySet(),
-            null,
-            verifyingClient,
-            new NoneCircuitBreakerService(),
-            new UsageService(),
-            identityService
-        );
+        controller = new RestController(Collections.emptySet(), null, verifyingClient, new NoneCircuitBreakerService(), new UsageService());
     }
 
     @After
diff --git a/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java b/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java
index 37df90fb103a3..d1e1a3e8af17c 100644
--- a/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java
+++ b/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java
@@ -123,6 +123,16 @@ public void disconnectFromNode(DiscoveryNode node) {
         delegate.disconnectFromNode(node);
     }
 
+    @Override
+    public void setPendingDisconnection(DiscoveryNode node) {
+        delegate.setPendingDisconnection(node);
+    }
+
+    @Override
+    public void clearPendingDisconnections() {
+        delegate.clearPendingDisconnections();
+    }
+
     @Override
     public int size() {
         return delegate.size();