From fcd7476d018ff55debd5b0b72b9e12d24c2cf998 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 28 Sep 2023 16:04:51 -0700
Subject: [PATCH 001/120] Implemented and tested time took for
 QuerySearchResult objects for use in tiered caching

---
 .idea/runConfigurations/Debug_OpenSearch.xml  |  6 +-
 .../opensearch/search/query/QueryPhase.java   |  3 +-
 .../search/query/QuerySearchResult.java       | 16 ++++
 .../opensearch/search/SearchServiceTests.java | 93 +++++++++++++++++++
 .../search/query/QueryPhaseTests.java         | 28 ++++++
 .../search/query/QuerySearchResultTests.java  |  4 +
 6 files changed, 148 insertions(+), 2 deletions(-)

diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml
index 0d8bf59823acf..c18046f873477 100644
--- a/.idea/runConfigurations/Debug_OpenSearch.xml
+++ b/.idea/runConfigurations/Debug_OpenSearch.xml
@@ -6,6 +6,10 @@
     <option name="HOST" value="localhost" />
     <option name="PORT" value="5005" />
     <option name="AUTO_RESTART" value="true" />
+    <RunnerSettings RunnerId="Debug">
+      <option name="DEBUG_PORT" value="5005" />
+      <option name="LOCAL" value="false" />
+    </RunnerSettings>
     <method v="2" />
   </configuration>
-</component>
+</component>
\ No newline at end of file
diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhase.java b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
index f3cf2c13ecdef..afac6ba2b62c7 100644
--- a/server/src/main/java/org/opensearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
@@ -138,6 +138,7 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep
                     new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Lucene.EMPTY_SCORE_DOCS), Float.NaN),
                     new DocValueFormat[0]
                 );
+            searchContext.queryResult().setTookTimeNanos();
             return;
         }
 
@@ -292,7 +293,7 @@ static boolean executeInternal(SearchContext searchContext, QueryPhaseSearcher q
                     queryResult.nodeQueueSize(rExecutor.getCurrentQueueSize());
                     queryResult.serviceTimeEWMA((long) rExecutor.getTaskExecutionEWMA());
                 }
-
+                queryResult.setTookTimeNanos();
                 return shouldRescore;
             } finally {
                 // Search phase has finished, no longer need to check for timeout
diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
index 7de605a244d09..e0899994c16f0 100644
--- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
@@ -87,13 +87,17 @@ public final class QuerySearchResult extends SearchPhaseResult {
     private int nodeQueueSize = -1;
 
     private final boolean isNull;
+    private long startTimeNanos;
+    private long tookTimeNanos;
 
     public QuerySearchResult() {
         this(false);
+        this.startTimeNanos = System.nanoTime();
     }
 
     public QuerySearchResult(StreamInput in) throws IOException {
         super(in);
+        this.startTimeNanos = System.nanoTime();
         isNull = in.readBoolean();
         if (isNull == false) {
             ShardSearchContextId id = new ShardSearchContextId(in);
@@ -102,6 +106,7 @@ public QuerySearchResult(StreamInput in) throws IOException {
     }
 
     public QuerySearchResult(ShardSearchContextId contextId, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) {
+        this.startTimeNanos = System.nanoTime();
         this.contextId = contextId;
         setSearchShardTarget(shardTarget);
         isNull = false;
@@ -364,6 +369,7 @@ public void readFromWithId(ShardSearchContextId id, StreamInput in) throws IOExc
         nodeQueueSize = in.readInt();
         setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new));
         setRescoreDocIds(new RescoreDocIds(in));
+        tookTimeNanos = in.readVLong();
     }
 
     @Override
@@ -406,6 +412,8 @@ public void writeToNoId(StreamOutput out) throws IOException {
         out.writeInt(nodeQueueSize);
         out.writeOptionalWriteable(getShardSearchRequest());
         getRescoreDocIds().writeTo(out);
+        setTookTimeNanos(); // put at end to capture as much of the time as possible
+        out.writeVLong(tookTimeNanos); // VLong as took time should always be positive
     }
 
     public TotalHits getTotalHits() {
@@ -415,4 +423,12 @@ public TotalHits getTotalHits() {
     public float getMaxScore() {
         return maxScore;
     }
+
+    public long getTookTimeNanos() {
+        return tookTimeNanos;
+    }
+
+    public void setTookTimeNanos() {
+        tookTimeNanos = System.nanoTime() - startTimeNanos;
+    }
 }
diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
index 7c84078af080e..a6ab68ee4b09f 100644
--- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
@@ -823,6 +823,96 @@ public Scroll scroll() {
         }
     }
 
+    public void testQuerySearchResultTookTime() throws Exception {
+        // This tests the code path when indicesService.canCache(request, context) is false
+        // I wasn't able to introduce a delay in these tests as everything between creation and usage of the QuerySearchResult object
+        // happen in a single line - we would have to modify executeQueryPhase to take a delay parameter
+        // However this was tested in QueryPhaseTests.java
+        createIndex("index");
+        final SearchService service = getInstanceFromNode(SearchService.class);
+        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
+        final IndexShard indexShard = indexService.getShard(0);
+        SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
+        searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
+
+        ShardSearchRequest request = new ShardSearchRequest(
+            OriginalIndices.NONE,
+            searchRequest,
+            indexShard.shardId(),
+            2, // must have >1 shards for executeQueryPhase to return the QuerySearchResult
+            new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1.0f,
+            -1,
+            null,
+            null
+        );
+
+        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
+        service.executeQueryPhase(request, randomBoolean(), task, new ActionListener<SearchPhaseResult>() {
+            @Override
+            public void onResponse(SearchPhaseResult searchPhaseResult) {
+                assertEquals(QuerySearchResult.class, searchPhaseResult.getClass()); // 2+ shards -> QuerySearchResult returned
+                QuerySearchResult qsr = (QuerySearchResult) searchPhaseResult;
+                assertTrue(qsr.getTookTimeNanos() > 0); // Above zero means it's been set at some point
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+        });
+    }
+    public void testQuerySearchResultTookTimeCacheableRequest() throws Exception {
+        // Test when the request is cacheable
+        // Similarly, no delay could be added
+        createIndex("index");
+        final SearchService service = getInstanceFromNode(SearchService.class);
+        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
+        final IndexShard indexShard = indexService.getShard(0);
+        SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+
+        searchRequest.source(searchSourceBuilder);
+        searchSourceBuilder.scriptField(
+            "field" + 0,
+            new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap())
+        );
+        searchSourceBuilder.size(0); // from testIgnoreScriptfieldIfSizeZero
+
+        String[] dummyRoutings = new String[]{};
+        OriginalIndices dummyOriginalIndices = new OriginalIndices(new String[]{"index'"}, IndicesOptions.LENIENT_EXPAND_OPEN);
+
+        ShardSearchRequest request = new ShardSearchRequest(
+            dummyOriginalIndices,
+            searchRequest,
+            indexShard.shardId(),
+            2, // must have >1 shards for executeQueryPhase to return the QuerySearchResult
+            new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1.0f,
+            0L,
+            // if nowInMillis is negative, it fails when trying to write the shardSearchRequest to cache as it uses WriteVLong which only takes positive longs
+            null,
+            dummyRoutings // similar for routings
+        );
+
+        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
+        service.executeQueryPhase(request, randomBoolean(), task, new ActionListener<SearchPhaseResult>() {
+            @Override
+            public void onResponse(SearchPhaseResult searchPhaseResult) {
+                assertEquals(QuerySearchResult.class, searchPhaseResult.getClass()); // 2+ shards -> QuerySearchResult returned
+                QuerySearchResult qsr = (QuerySearchResult) searchPhaseResult;
+                assertTrue(qsr.getTookTimeNanos() > 0); // Above zero means it's been set at some point
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+        });
+    }
+
     public void testCanMatch() throws Exception {
         createIndex("index");
         final SearchService service = getInstanceFromNode(SearchService.class);
@@ -996,6 +1086,8 @@ public void onResponse(SearchPhaseResult searchPhaseResult) {
                 try {
                     // make sure that the wrapper is called when the query is actually executed
                     assertEquals(1, numWrapInvocations.get());
+                    System.out.println(searchPhaseResult.getClass());
+                    //System.out.println(((QuerySearchResult) searchPhaseResult).getTookTimeNanos());
                 } finally {
                     latch.countDown();
                 }
@@ -1010,6 +1102,7 @@ public void onFailure(Exception e) {
                 }
             }
         });
+
         latch.await();
     }
 
diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
index 39126a607f968..70159a094d03d 100644
--- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
@@ -277,6 +277,34 @@ public void testPostFilterDisablesCountOptimization() throws Exception {
         dir.close();
     }
 
+    public void testQueryTookTime() throws Exception {
+        Directory dir = newDirectory();
+        final Sort sort = new Sort(new SortField("rank", SortField.Type.INT));
+        IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort);
+        RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+        Document doc = new Document();
+        for (int i = 0; i < 10; i++) {
+            doc.add(new StringField("foo", Integer.toString(i), Store.NO));
+        }
+        w.addDocument(doc);
+        w.close();
+        IndexReader reader = DirectoryReader.open(dir);
+
+        TestSearchContext context = new TestSearchContext(null, indexShard, newEarlyTerminationContextSearcher(reader, 0, executor));
+        // QuerySearchResponse object is initialized in above line, so tookTime counts from here
+        context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()));
+        context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
+
+        int sleepTimeMillis = 1000;
+        Thread.sleep(sleepTimeMillis); // wait one second and ensure tookTime is greater than this amount
+
+        QueryPhase.executeInternal(context.withCleanQueryResult(), queryPhaseSearcher);
+        long tookTime = context.queryResult().getTookTimeNanos();
+        assertTrue(tookTime > 1000000 * sleepTimeMillis);
+        reader.close();
+        dir.close();
+    }
+
     public void testTerminateAfterWithFilter() throws Exception {
         Directory dir = newDirectory();
         final Sort sort = new Sort(new SortField("rank", SortField.Type.INT));
diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
index 41e4e1ae45a73..2932382c9f3b5 100644
--- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
@@ -99,6 +99,9 @@ private static QuerySearchResult createTestInstance() throws Exception {
         if (randomBoolean()) {
             result.aggregations(InternalAggregationsTests.createTestInstance());
         }
+        assertEquals(0, result.getTookTimeNanos());
+        result.setTookTimeNanos();
+        assertTrue(result.getTookTimeNanos() > 0);
         return result;
     }
 
@@ -118,6 +121,7 @@ public void testSerialization() throws Exception {
             assertEquals(aggs.asList(), deserializedAggs.asList());
         }
         assertEquals(querySearchResult.terminatedEarly(), deserialized.terminatedEarly());
+        assertEquals(querySearchResult.getTookTimeNanos(), deserialized.getTookTimeNanos());
     }
 
     public void testNullResponse() throws Exception {

From 87cbe2a77f67a63287313ceffe3652148c8e4691 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 2 Oct 2023 12:45:08 -0700
Subject: [PATCH 002/120] Moved setting to QueryPhase.execute()

---
 .../opensearch/search/query/QueryPhase.java   |  5 +--
 .../search/query/QuerySearchResult.java       |  9 ++----
 .../opensearch/search/SearchServiceTests.java | 32 ++++++++++++++++---
 .../search/query/QueryPhaseTests.java         | 28 ----------------
 .../search/query/QuerySearchResultTests.java  |  2 --
 5 files changed, 32 insertions(+), 44 deletions(-)

diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhase.java b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
index afac6ba2b62c7..14b239d3270d4 100644
--- a/server/src/main/java/org/opensearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
@@ -131,6 +131,7 @@ public void preProcess(SearchContext context) {
     }
 
     public void execute(SearchContext searchContext) throws QueryPhaseExecutionException {
+        long startTime = System.nanoTime();
         if (searchContext.hasOnlySuggest()) {
             suggestProcessor.process(searchContext);
             searchContext.queryResult()
@@ -138,7 +139,7 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep
                     new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Lucene.EMPTY_SCORE_DOCS), Float.NaN),
                     new DocValueFormat[0]
                 );
-            searchContext.queryResult().setTookTimeNanos();
+            searchContext.queryResult().setTookTimeNanos(System.nanoTime() - startTime);
             return;
         }
 
@@ -166,6 +167,7 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep
             );
             searchContext.queryResult().profileResults(shardResults);
         }
+        searchContext.queryResult().setTookTimeNanos(System.nanoTime() - startTime);
     }
 
     // making public for testing
@@ -293,7 +295,6 @@ static boolean executeInternal(SearchContext searchContext, QueryPhaseSearcher q
                     queryResult.nodeQueueSize(rExecutor.getCurrentQueueSize());
                     queryResult.serviceTimeEWMA((long) rExecutor.getTaskExecutionEWMA());
                 }
-                queryResult.setTookTimeNanos();
                 return shouldRescore;
             } finally {
                 // Search phase has finished, no longer need to check for timeout
diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
index e0899994c16f0..5a8cf06bd8b0a 100644
--- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
@@ -87,17 +87,14 @@ public final class QuerySearchResult extends SearchPhaseResult {
     private int nodeQueueSize = -1;
 
     private final boolean isNull;
-    private long startTimeNanos;
     private long tookTimeNanos;
 
     public QuerySearchResult() {
         this(false);
-        this.startTimeNanos = System.nanoTime();
     }
 
     public QuerySearchResult(StreamInput in) throws IOException {
         super(in);
-        this.startTimeNanos = System.nanoTime();
         isNull = in.readBoolean();
         if (isNull == false) {
             ShardSearchContextId id = new ShardSearchContextId(in);
@@ -106,7 +103,6 @@ public QuerySearchResult(StreamInput in) throws IOException {
     }
 
     public QuerySearchResult(ShardSearchContextId contextId, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) {
-        this.startTimeNanos = System.nanoTime();
         this.contextId = contextId;
         setSearchShardTarget(shardTarget);
         isNull = false;
@@ -412,7 +408,6 @@ public void writeToNoId(StreamOutput out) throws IOException {
         out.writeInt(nodeQueueSize);
         out.writeOptionalWriteable(getShardSearchRequest());
         getRescoreDocIds().writeTo(out);
-        setTookTimeNanos(); // put at end to capture as much of the time as possible
         out.writeVLong(tookTimeNanos); // VLong as took time should always be positive
     }
 
@@ -428,7 +423,7 @@ public long getTookTimeNanos() {
         return tookTimeNanos;
     }
 
-    public void setTookTimeNanos() {
-        tookTimeNanos = System.nanoTime() - startTimeNanos;
+    public void setTookTimeNanos(long tookTime) {
+        tookTimeNanos = tookTime;
     }
 }
diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
index a6ab68ee4b09f..3af22e43106a4 100644
--- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
@@ -54,6 +54,7 @@
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.common.UUIDs;
+import org.opensearch.common.recycler.Recycler;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.unit.TimeValue;
@@ -121,6 +122,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Semaphore;
@@ -824,10 +826,9 @@ public Scroll scroll() {
     }
 
     public void testQuerySearchResultTookTime() throws Exception {
-        // This tests the code path when indicesService.canCache(request, context) is false
         // I wasn't able to introduce a delay in these tests as everything between creation and usage of the QuerySearchResult object
-        // happen in a single line - we would have to modify executeQueryPhase to take a delay parameter
-        // However this was tested in QueryPhaseTests.java
+        // happen in a single line - we would have to modify QueryPhase.execute() to take a delay parameter
+        // However this was tested manually
         createIndex("index");
         final SearchService service = getInstanceFromNode(SearchService.class);
         final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
@@ -864,7 +865,7 @@ public void onFailure(Exception e) {
         });
     }
     public void testQuerySearchResultTookTimeCacheableRequest() throws Exception {
-        // Test when the request is cacheable
+        // Test 2 identical cacheable requests and assert both have the same tookTime
         // Similarly, no delay could be added
         createIndex("index");
         final SearchService service = getInstanceFromNode(SearchService.class);
@@ -897,13 +898,15 @@ public void testQuerySearchResultTookTimeCacheableRequest() throws Exception {
             dummyRoutings // similar for routings
         );
 
+        final CompletableFuture<Long> firstResult = new CompletableFuture<>();
+        final CompletableFuture<Long> secondResult = new CompletableFuture<>();
         SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
         service.executeQueryPhase(request, randomBoolean(), task, new ActionListener<SearchPhaseResult>() {
             @Override
             public void onResponse(SearchPhaseResult searchPhaseResult) {
                 assertEquals(QuerySearchResult.class, searchPhaseResult.getClass()); // 2+ shards -> QuerySearchResult returned
                 QuerySearchResult qsr = (QuerySearchResult) searchPhaseResult;
-                assertTrue(qsr.getTookTimeNanos() > 0); // Above zero means it's been set at some point
+                firstResult.complete(qsr.getTookTimeNanos());
             }
 
             @Override
@@ -911,6 +914,25 @@ public void onFailure(Exception e) {
                 throw new AssertionError(e);
             }
         });
+
+        service.executeQueryPhase(request, randomBoolean(), task, new ActionListener<SearchPhaseResult>() {
+            @Override
+            public void onResponse(SearchPhaseResult searchPhaseResult) {
+                assertEquals(QuerySearchResult.class, searchPhaseResult.getClass()); // 2+ shards -> QuerySearchResult returned
+                QuerySearchResult qsr = (QuerySearchResult) searchPhaseResult;
+                secondResult.complete(qsr.getTookTimeNanos());
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+        });
+
+        long firstResultVal = firstResult.get();
+        long secondResultVal = secondResult.get();
+        assertEquals(firstResultVal, secondResultVal);
+        assertTrue(firstResultVal > 0);
     }
 
     public void testCanMatch() throws Exception {
diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
index 70159a094d03d..39126a607f968 100644
--- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
@@ -277,34 +277,6 @@ public void testPostFilterDisablesCountOptimization() throws Exception {
         dir.close();
     }
 
-    public void testQueryTookTime() throws Exception {
-        Directory dir = newDirectory();
-        final Sort sort = new Sort(new SortField("rank", SortField.Type.INT));
-        IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort);
-        RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
-        Document doc = new Document();
-        for (int i = 0; i < 10; i++) {
-            doc.add(new StringField("foo", Integer.toString(i), Store.NO));
-        }
-        w.addDocument(doc);
-        w.close();
-        IndexReader reader = DirectoryReader.open(dir);
-
-        TestSearchContext context = new TestSearchContext(null, indexShard, newEarlyTerminationContextSearcher(reader, 0, executor));
-        // QuerySearchResponse object is initialized in above line, so tookTime counts from here
-        context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()));
-        context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
-
-        int sleepTimeMillis = 1000;
-        Thread.sleep(sleepTimeMillis); // wait one second and ensure tookTime is greater than this amount
-
-        QueryPhase.executeInternal(context.withCleanQueryResult(), queryPhaseSearcher);
-        long tookTime = context.queryResult().getTookTimeNanos();
-        assertTrue(tookTime > 1000000 * sleepTimeMillis);
-        reader.close();
-        dir.close();
-    }
-
     public void testTerminateAfterWithFilter() throws Exception {
         Directory dir = newDirectory();
         final Sort sort = new Sort(new SortField("rank", SortField.Type.INT));
diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
index 2932382c9f3b5..0d83777dbd4cf 100644
--- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
@@ -100,8 +100,6 @@ private static QuerySearchResult createTestInstance() throws Exception {
             result.aggregations(InternalAggregationsTests.createTestInstance());
         }
         assertEquals(0, result.getTookTimeNanos());
-        result.setTookTimeNanos();
-        assertTrue(result.getTookTimeNanos() > 0);
         return result;
     }
 

From cf18eb9c31434c933ee73e20a3796bffbc6c47fc Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 2 Oct 2023 12:45:08 -0700
Subject: [PATCH 003/120] Moved setting to QueryPhase.execute()

---
 .../src/test/java/org/opensearch/search/SearchServiceTests.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
index 3af22e43106a4..caef2a450044f 100644
--- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
@@ -1108,8 +1108,6 @@ public void onResponse(SearchPhaseResult searchPhaseResult) {
                 try {
                     // make sure that the wrapper is called when the query is actually executed
                     assertEquals(1, numWrapInvocations.get());
-                    System.out.println(searchPhaseResult.getClass());
-                    //System.out.println(((QuerySearchResult) searchPhaseResult).getTookTimeNanos());
                 } finally {
                     latch.countDown();
                 }

From c876de81f2cf00ab1a4295abfaef0466042331da Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 3 Oct 2023 14:44:15 -0700
Subject: [PATCH 004/120] spotlessCheck

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../java/org/opensearch/search/SearchServiceTests.java   | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
index caef2a450044f..f4657e303fbd8 100644
--- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
@@ -54,7 +54,6 @@
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.common.UUIDs;
-import org.opensearch.common.recycler.Recycler;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.unit.TimeValue;
@@ -864,6 +863,7 @@ public void onFailure(Exception e) {
             }
         });
     }
+
     public void testQuerySearchResultTookTimeCacheableRequest() throws Exception {
         // Test 2 identical cacheable requests and assert both have the same tookTime
         // Similarly, no delay could be added
@@ -882,8 +882,8 @@ public void testQuerySearchResultTookTimeCacheableRequest() throws Exception {
         );
         searchSourceBuilder.size(0); // from testIgnoreScriptfieldIfSizeZero
 
-        String[] dummyRoutings = new String[]{};
-        OriginalIndices dummyOriginalIndices = new OriginalIndices(new String[]{"index'"}, IndicesOptions.LENIENT_EXPAND_OPEN);
+        String[] dummyRoutings = new String[] {};
+        OriginalIndices dummyOriginalIndices = new OriginalIndices(new String[] { "index'" }, IndicesOptions.LENIENT_EXPAND_OPEN);
 
         ShardSearchRequest request = new ShardSearchRequest(
             dummyOriginalIndices,
@@ -893,7 +893,8 @@ public void testQuerySearchResultTookTimeCacheableRequest() throws Exception {
             new AliasFilter(null, Strings.EMPTY_ARRAY),
             1.0f,
             0L,
-            // if nowInMillis is negative, it fails when trying to write the shardSearchRequest to cache as it uses WriteVLong which only takes positive longs
+            // if nowInMillis is negative, it fails when trying to write the shardSearchRequest to cache as it uses WriteVLong which only
+            // takes positive longs
             null,
             dummyRoutings // similar for routings
         );

From 93a16d4a3e24acb785e7e9f6651a2ad0453acbff Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 4 Oct 2023 13:24:57 -0700
Subject: [PATCH 005/120] Added first draft of caching policy setup

---
 server/src/main/java/org/opensearch/indices/CacheTierPolicy.java  | 0
 server/src/main/java/org/opensearch/indices/CheckDataResult.java  | 0
 server/src/main/java/org/opensearch/indices/DummyIRCPolicy.java   | 0
 .../org/opensearch/indices/IndicesRequestCacheDiskTierPolicy.java | 0
 .../org/opensearch/indices/IndicesRequestCacheTookTimePolicy.java | 0
 5 files changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/indices/CacheTierPolicy.java
 create mode 100644 server/src/main/java/org/opensearch/indices/CheckDataResult.java
 create mode 100644 server/src/main/java/org/opensearch/indices/DummyIRCPolicy.java
 create mode 100644 server/src/main/java/org/opensearch/indices/IndicesRequestCacheDiskTierPolicy.java
 create mode 100644 server/src/main/java/org/opensearch/indices/IndicesRequestCacheTookTimePolicy.java

diff --git a/server/src/main/java/org/opensearch/indices/CacheTierPolicy.java b/server/src/main/java/org/opensearch/indices/CacheTierPolicy.java
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/server/src/main/java/org/opensearch/indices/CheckDataResult.java b/server/src/main/java/org/opensearch/indices/CheckDataResult.java
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/server/src/main/java/org/opensearch/indices/DummyIRCPolicy.java b/server/src/main/java/org/opensearch/indices/DummyIRCPolicy.java
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCacheDiskTierPolicy.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCacheDiskTierPolicy.java
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCacheTookTimePolicy.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCacheTookTimePolicy.java
new file mode 100644
index 0000000000000..e69de29bb2d1d

From 8d6c8c49f014ed8539db66f2d907c71b867ed6cc Mon Sep 17 00:00:00 2001
From: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>
Date: Mon, 9 Oct 2023 08:34:15 -0700
Subject: [PATCH 006/120] Tiered caching framework draft changes

---
 .../org/opensearch/common/cache/Cache.java    | 112 ++++++------
 .../common/cache/RemovalNotification.java     |  12 ++
 .../cache/request/ShardRequestCache.java      |  48 +++--
 .../AbstractIndexShardCacheEntity.java        |  19 +-
 .../org/opensearch/indices/CachingTier.java   |  39 ++++
 .../opensearch/indices/DiskCachingTier.java   |  13 ++
 .../indices/DummyDiskCachingTier.java         |  58 ++++++
 .../indices/IndicesRequestCache.java          |  83 ++++++---
 .../opensearch/indices/OnHeapCachingTier.java |  11 ++
 .../indices/OpenSearchOnHeapCache.java        | 119 ++++++++++++
 .../java/org/opensearch/indices/TierType.java |  15 ++
 .../indices/TieredCacheEventListener.java     |  22 +++
 .../indices/TieredCacheHandler.java           |  24 +++
 .../opensearch/indices/TieredCacheLoader.java |  15 ++
 .../TieredCacheSpilloverStrategyHandler.java  | 172 ++++++++++++++++++
 .../indices/IndicesServiceCloseTests.java     |   6 +-
 16 files changed, 661 insertions(+), 107 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/indices/CachingTier.java
 create mode 100644 server/src/main/java/org/opensearch/indices/DiskCachingTier.java
 create mode 100644 server/src/main/java/org/opensearch/indices/DummyDiskCachingTier.java
 create mode 100644 server/src/main/java/org/opensearch/indices/OnHeapCachingTier.java
 create mode 100644 server/src/main/java/org/opensearch/indices/OpenSearchOnHeapCache.java
 create mode 100644 server/src/main/java/org/opensearch/indices/TierType.java
 create mode 100644 server/src/main/java/org/opensearch/indices/TieredCacheEventListener.java
 create mode 100644 server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
 create mode 100644 server/src/main/java/org/opensearch/indices/TieredCacheLoader.java
 create mode 100644 server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java

diff --git a/server/src/main/java/org/opensearch/common/cache/Cache.java b/server/src/main/java/org/opensearch/common/cache/Cache.java
index 0ebef1556424b..c7c28e13496e5 100644
--- a/server/src/main/java/org/opensearch/common/cache/Cache.java
+++ b/server/src/main/java/org/opensearch/common/cache/Cache.java
@@ -422,68 +422,74 @@ public V computeIfAbsent(K key, CacheLoader<K, V> loader) throws ExecutionExcept
             }
         });
         if (value == null) {
-            // we need to synchronize loading of a value for a given key; however, holding the segment lock while
-            // invoking load can lead to deadlock against another thread due to dependent key loading; therefore, we
-            // need a mechanism to ensure that load is invoked at most once, but we are not invoking load while holding
-            // the segment lock; to do this, we atomically put a future in the map that can load the value, and then
-            // get the value from this future on the thread that won the race to place the future into the segment map
-            CacheSegment<K, V> segment = getCacheSegment(key);
-            CompletableFuture<Entry<K, V>> future;
-            CompletableFuture<Entry<K, V>> completableFuture = new CompletableFuture<>();
+            value = compute(key, loader);
+        }
+        return value;
+    }
 
-            try (ReleasableLock ignored = segment.writeLock.acquire()) {
-                future = segment.map.putIfAbsent(key, completableFuture);
-            }
+    public V compute(K key, CacheLoader<K, V> loader) throws ExecutionException {
+        long now = now();
+        // we need to synchronize loading of a value for a given key; however, holding the segment lock while
+        // invoking load can lead to deadlock against another thread due to dependent key loading; therefore, we
+        // need a mechanism to ensure that load is invoked at most once, but we are not invoking load while holding
+        // the segment lock; to do this, we atomically put a future in the map that can load the value, and then
+        // get the value from this future on the thread that won the race to place the future into the segment map
+        CacheSegment<K, V> segment = getCacheSegment(key);
+        CompletableFuture<Entry<K, V>> future;
+        CompletableFuture<Entry<K, V>> completableFuture = new CompletableFuture<>();
 
-            BiFunction<? super Entry<K, V>, Throwable, ? extends V> handler = (ok, ex) -> {
-                if (ok != null) {
-                    try (ReleasableLock ignored = lruLock.acquire()) {
-                        promote(ok, now);
-                    }
-                    return ok.value;
-                } else {
-                    try (ReleasableLock ignored = segment.writeLock.acquire()) {
-                        CompletableFuture<Entry<K, V>> sanity = segment.map.get(key);
-                        if (sanity != null && sanity.isCompletedExceptionally()) {
-                            segment.map.remove(key);
-                        }
-                    }
-                    return null;
-                }
-            };
+        try (ReleasableLock ignored = segment.writeLock.acquire()) {
+            future = segment.map.putIfAbsent(key, completableFuture);
+        }
 
-            CompletableFuture<V> completableValue;
-            if (future == null) {
-                future = completableFuture;
-                completableValue = future.handle(handler);
-                V loaded;
-                try {
-                    loaded = loader.load(key);
-                } catch (Exception e) {
-                    future.completeExceptionally(e);
-                    throw new ExecutionException(e);
-                }
-                if (loaded == null) {
-                    NullPointerException npe = new NullPointerException("loader returned a null value");
-                    future.completeExceptionally(npe);
-                    throw new ExecutionException(npe);
-                } else {
-                    future.complete(new Entry<>(key, loaded, now));
+        BiFunction<? super Entry<K, V>, Throwable, ? extends V> handler = (ok, ex) -> {
+            if (ok != null) {
+                try (ReleasableLock ignored = lruLock.acquire()) {
+                    promote(ok, now);
                 }
+                return ok.value;
             } else {
-                completableValue = future.handle(handler);
+                try (ReleasableLock ignored = segment.writeLock.acquire()) {
+                    CompletableFuture<Entry<K, V>> sanity = segment.map.get(key);
+                    if (sanity != null && sanity.isCompletedExceptionally()) {
+                        segment.map.remove(key);
+                    }
+                }
+                return null;
             }
+        };
 
+        CompletableFuture<V> completableValue;
+        if (future == null) {
+            future = completableFuture;
+            completableValue = future.handle(handler);
+            V loaded;
             try {
-                value = completableValue.get();
-                // check to ensure the future hasn't been completed with an exception
-                if (future.isCompletedExceptionally()) {
-                    future.get(); // call get to force the exception to be thrown for other concurrent callers
-                    throw new IllegalStateException("the future was completed exceptionally but no exception was thrown");
-                }
-            } catch (InterruptedException e) {
-                throw new IllegalStateException(e);
+                loaded = loader.load(key);
+            } catch (Exception e) {
+                future.completeExceptionally(e);
+                throw new ExecutionException(e);
             }
+            if (loaded == null) {
+                NullPointerException npe = new NullPointerException("loader returned a null value");
+                future.completeExceptionally(npe);
+                throw new ExecutionException(npe);
+            } else {
+                future.complete(new Entry<>(key, loaded, now));
+            }
+        } else {
+            completableValue = future.handle(handler);
+        }
+        V value;
+        try {
+            value = completableValue.get();
+            // check to ensure the future hasn't been completed with an exception
+            if (future.isCompletedExceptionally()) {
+                future.get(); // call get to force the exception to be thrown for other concurrent callers
+                throw new IllegalStateException("the future was completed exceptionally but no exception was thrown");
+            }
+        } catch (InterruptedException e) {
+            throw new IllegalStateException(e);
         }
         return value;
     }
diff --git a/server/src/main/java/org/opensearch/common/cache/RemovalNotification.java b/server/src/main/java/org/opensearch/common/cache/RemovalNotification.java
index 6d355b2122460..71e240064c6ae 100644
--- a/server/src/main/java/org/opensearch/common/cache/RemovalNotification.java
+++ b/server/src/main/java/org/opensearch/common/cache/RemovalNotification.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.common.cache;
 
+import org.opensearch.indices.TierType;
+
 /**
  * Notification when an element is removed from the cache
  *
@@ -42,11 +44,17 @@ public class RemovalNotification<K, V> {
     private final K key;
     private final V value;
     private final RemovalReason removalReason;
+    private final TierType tierType;
 
     public RemovalNotification(K key, V value, RemovalReason removalReason) {
+        this(key, value, removalReason, TierType.ON_HEAP);
+    }
+
+    public RemovalNotification(K key, V value, RemovalReason removalReason, TierType tierType) {
         this.key = key;
         this.value = value;
         this.removalReason = removalReason;
+        this.tierType = tierType;
     }
 
     public K getKey() {
@@ -60,4 +68,8 @@ public V getValue() {
     public RemovalReason getRemovalReason() {
         return removalReason;
     }
+
+    public TierType getTierType() {
+        return tierType;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index b13eec79c2be8..1beef5217355f 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -35,6 +35,9 @@
 import org.apache.lucene.util.Accountable;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.indices.TierType;
+
+import java.util.EnumMap;
 
 /**
  * Tracks the portion of the request cache in use for a particular shard.
@@ -43,30 +46,39 @@
  */
 public final class ShardRequestCache {
 
-    final CounterMetric evictionsMetric = new CounterMetric();
-    final CounterMetric totalMetric = new CounterMetric();
-    final CounterMetric hitCount = new CounterMetric();
-    final CounterMetric missCount = new CounterMetric();
+    private EnumMap<TierType, StatsHolder> statsHolder = new EnumMap<>(TierType.class);
+
+    public ShardRequestCache() {
+        for (TierType tierType : TierType.values()) {
+            statsHolder.put(tierType, new StatsHolder());
+        }
+    }
 
     public RequestCacheStats stats() {
-        return new RequestCacheStats(totalMetric.count(), evictionsMetric.count(), hitCount.count(), missCount.count());
+        // TODO: Change RequestCacheStats to support disk tier stats.
+        return new RequestCacheStats(
+            statsHolder.get(TierType.ON_HEAP).totalMetric.count(),
+            statsHolder.get(TierType.ON_HEAP).evictionsMetric.count(),
+            statsHolder.get(TierType.ON_HEAP).hitCount.count(),
+            statsHolder.get(TierType.ON_HEAP).missCount.count()
+        );
     }
 
-    public void onHit() {
-        hitCount.inc();
+    public void onHit(TierType tierType) {
+        statsHolder.get(tierType).hitCount.inc();
     }
 
-    public void onMiss() {
-        missCount.inc();
+    public void onMiss(TierType tierType) {
+        statsHolder.get(tierType).missCount.inc();
     }
 
-    public void onCached(Accountable key, BytesReference value) {
-        totalMetric.inc(key.ramBytesUsed() + value.ramBytesUsed());
+    public void onCached(Accountable key, BytesReference value, TierType tierType) {
+        statsHolder.get(tierType).totalMetric.inc(key.ramBytesUsed() + value.ramBytesUsed());
     }
 
-    public void onRemoval(Accountable key, BytesReference value, boolean evicted) {
+    public void onRemoval(Accountable key, BytesReference value, boolean evicted, TierType tierType) {
         if (evicted) {
-            evictionsMetric.inc();
+            statsHolder.get(tierType).evictionsMetric.inc();
         }
         long dec = 0;
         if (key != null) {
@@ -75,6 +87,14 @@ public void onRemoval(Accountable key, BytesReference value, boolean evicted) {
         if (value != null) {
             dec += value.ramBytesUsed();
         }
-        totalMetric.dec(dec);
+        statsHolder.get(tierType).totalMetric.dec(dec);
+    }
+
+    static class StatsHolder {
+
+        final CounterMetric evictionsMetric = new CounterMetric();
+        final CounterMetric totalMetric = new CounterMetric();
+        final CounterMetric hitCount = new CounterMetric();
+        final CounterMetric missCount = new CounterMetric();
     }
 }
diff --git a/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java b/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
index bb1201cb910a9..2eef16df2bb9a 100644
--- a/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
+++ b/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
@@ -51,22 +51,27 @@ abstract class AbstractIndexShardCacheEntity implements IndicesRequestCache.Cach
     protected abstract ShardRequestCache stats();
 
     @Override
-    public final void onCached(IndicesRequestCache.Key key, BytesReference value) {
-        stats().onCached(key, value);
+    public final void onCached(IndicesRequestCache.Key key, BytesReference value, TierType tierType) {
+        stats().onCached(key, value, tierType);
     }
 
     @Override
-    public final void onHit() {
-        stats().onHit();
+    public final void onHit(TierType tierType) {
+        stats().onHit(tierType);
     }
 
     @Override
-    public final void onMiss() {
-        stats().onMiss();
+    public final void onMiss(TierType tierType) {
+        stats().onMiss(tierType);
     }
 
     @Override
     public final void onRemoval(RemovalNotification<IndicesRequestCache.Key, BytesReference> notification) {
-        stats().onRemoval(notification.getKey(), notification.getValue(), notification.getRemovalReason() == RemovalReason.EVICTED);
+        stats().onRemoval(
+            notification.getKey(),
+            notification.getValue(),
+            notification.getRemovalReason() == RemovalReason.EVICTED,
+            notification.getTierType()
+        );
     }
 }
diff --git a/server/src/main/java/org/opensearch/indices/CachingTier.java b/server/src/main/java/org/opensearch/indices/CachingTier.java
new file mode 100644
index 0000000000000..85596929cfd6b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/CachingTier.java
@@ -0,0 +1,39 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.cache.RemovalListener;
+
+/**
+ * asdsadssa
+ * @param <K>
+ * @param <V>
+ */
+public interface CachingTier<K, V> {
+
+    V get(K key);
+
+    void put(K key, V value);
+
+    V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception;
+
+    void invalidate(K key);
+
+    V compute(K key, TieredCacheLoader<K, V> loader) throws Exception;
+
+    void setRemovalListener(RemovalListener<K, V> removalListener);
+
+    void invalidateAll();
+
+    Iterable<K> keys();
+
+    int count();
+
+    TierType getTierType();
+}
diff --git a/server/src/main/java/org/opensearch/indices/DiskCachingTier.java b/server/src/main/java/org/opensearch/indices/DiskCachingTier.java
new file mode 100644
index 0000000000000..efd9a459cd338
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/DiskCachingTier.java
@@ -0,0 +1,13 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+public interface DiskCachingTier<K, V> extends CachingTier<K, V> {
+
+}
diff --git a/server/src/main/java/org/opensearch/indices/DummyDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/DummyDiskCachingTier.java
new file mode 100644
index 0000000000000..26a78b6c61920
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/DummyDiskCachingTier.java
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.cache.RemovalListener;
+
+import java.util.Collections;
+
+public class DummyDiskCachingTier<K, V> implements CachingTier<K, V> {
+
+    @Override
+    public V get(K key) {
+        return null;
+    }
+
+    @Override
+    public void put(K key, V value) {}
+
+    @Override
+    public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception {
+        return null;
+    }
+
+    @Override
+    public void invalidate(K key) {}
+
+    @Override
+    public V compute(K key, TieredCacheLoader<K, V> loader) throws Exception {
+        return null;
+    }
+
+    @Override
+    public void setRemovalListener(RemovalListener<K, V> removalListener) {}
+
+    @Override
+    public void invalidateAll() {}
+
+    @Override
+    public Iterable<K> keys() {
+        return Collections::emptyIterator;
+    }
+
+    @Override
+    public int count() {
+        return 0;
+    }
+
+    @Override
+    public TierType getTierType() {
+        return TierType.DISK;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 629cea102a8b2..ab80e63a4abce 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -39,10 +39,6 @@
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.opensearch.common.CheckedSupplier;
-import org.opensearch.common.cache.Cache;
-import org.opensearch.common.cache.CacheBuilder;
-import org.opensearch.common.cache.CacheLoader;
-import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
 import org.opensearch.common.settings.Setting;
@@ -78,7 +74,7 @@
  *
  * @opensearch.internal
  */
-public final class IndicesRequestCache implements RemovalListener<IndicesRequestCache.Key, BytesReference>, Closeable {
+public final class IndicesRequestCache implements TieredCacheEventListener<IndicesRequestCache.Key, BytesReference>, Closeable {
 
     private static final Logger logger = LogManager.getLogger(IndicesRequestCache.class);
 
@@ -107,25 +103,35 @@ public final class IndicesRequestCache implements RemovalListener<IndicesRequest
     private final Set<CleanupKey> keysToClean = ConcurrentCollections.newConcurrentSet();
     private final ByteSizeValue size;
     private final TimeValue expire;
-    private final Cache<Key, BytesReference> cache;
+    // private final Cache<Key, BytesReference> cache;
+
+    private final TieredCacheHandler<Key, BytesReference> tieredCacheHandler;
 
     IndicesRequestCache(Settings settings) {
         this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
         this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
         long sizeInBytes = size.getBytes();
-        CacheBuilder<Key, BytesReference> cacheBuilder = CacheBuilder.<Key, BytesReference>builder()
-            .setMaximumWeight(sizeInBytes)
-            .weigher((k, v) -> k.ramBytesUsed() + v.ramBytesUsed())
-            .removalListener(this);
-        if (expire != null) {
-            cacheBuilder.setExpireAfterAccess(expire);
-        }
-        cache = cacheBuilder.build();
+        // CacheBuilder<Key, BytesReference> cacheBuilder = CacheBuilder.<Key, BytesReference>builder()
+        // .setMaximumWeight(sizeInBytes)
+        // .weigher((k, v) -> k.ramBytesUsed() + v.ramBytesUsed());
+        // //.removalListener(this);
+        // if (expire != null) {
+        // cacheBuilder.setExpireAfterAccess(expire);
+        // }
+        // cache = cacheBuilder.build();
+
+        OnHeapCachingTier<Key, BytesReference> openSearchOnHeapCache = new OpenSearchOnHeapCache.Builder<Key, BytesReference>().setWeigher(
+            (k, v) -> k.ramBytesUsed() + v.ramBytesUsed()
+        ).setMaximumWeight(sizeInBytes).setExpireAfterAccess(expire).build();
+
+        tieredCacheHandler = new TieredCacheSpilloverStrategyHandler.Builder<Key, BytesReference>().setOnHeapCachingTier(
+            openSearchOnHeapCache
+        ).setOnDiskCachingTier(new DummyDiskCachingTier<>()).setTieredCacheEventListener(this).build();
     }
 
     @Override
     public void close() {
-        cache.invalidateAll();
+        tieredCacheHandler.invalidateAll();
     }
 
     void clear(CacheEntity entity) {
@@ -133,11 +139,26 @@ void clear(CacheEntity entity) {
         cleanCache();
     }
 
+    @Override
+    public void onMiss(Key key, TierType tierType) {
+        key.entity.onMiss(tierType);
+    }
+
     @Override
     public void onRemoval(RemovalNotification<Key, BytesReference> notification) {
         notification.getKey().entity.onRemoval(notification);
     }
 
+    @Override
+    public void onHit(Key key, BytesReference value, TierType tierType) {
+        key.entity.onHit(tierType);
+    }
+
+    @Override
+    public void onCached(Key key, BytesReference value, TierType tierType) {
+        key.entity.onCached(key, value, tierType);
+    }
+
     BytesReference getOrCompute(
         CacheEntity cacheEntity,
         CheckedSupplier<BytesReference, IOException> loader,
@@ -147,9 +168,9 @@ BytesReference getOrCompute(
         assert reader.getReaderCacheHelper() != null;
         final Key key = new Key(cacheEntity, reader.getReaderCacheHelper().getKey(), cacheKey);
         Loader cacheLoader = new Loader(cacheEntity, loader);
-        BytesReference value = cache.computeIfAbsent(key, cacheLoader);
+        BytesReference value = tieredCacheHandler.computeIfAbsent(key, cacheLoader);
         if (cacheLoader.isLoaded()) {
-            key.entity.onMiss();
+            // key.entity.onMiss();
             // see if its the first time we see this reader, and make sure to register a cleanup key
             CleanupKey cleanupKey = new CleanupKey(cacheEntity, reader.getReaderCacheHelper().getKey());
             if (!registeredClosedListeners.containsKey(cleanupKey)) {
@@ -158,9 +179,10 @@ BytesReference getOrCompute(
                     OpenSearchDirectoryReader.addReaderCloseListener(reader, cleanupKey);
                 }
             }
-        } else {
-            key.entity.onHit();
         }
+        // else {
+        // key.entity.onHit();
+        // }
         return value;
     }
 
@@ -172,7 +194,8 @@ BytesReference getOrCompute(
      */
     void invalidate(CacheEntity cacheEntity, DirectoryReader reader, BytesReference cacheKey) {
         assert reader.getReaderCacheHelper() != null;
-        cache.invalidate(new Key(cacheEntity, reader.getReaderCacheHelper().getKey(), cacheKey));
+        tieredCacheHandler.invalidate(new Key(cacheEntity, reader.getReaderCacheHelper().getKey(), cacheKey));
+        // cache.invalidate(new Key(cacheEntity, reader.getReaderCacheHelper().getKey(), cacheKey));
     }
 
     /**
@@ -180,7 +203,7 @@ void invalidate(CacheEntity cacheEntity, DirectoryReader reader, BytesReference
      *
      * @opensearch.internal
      */
-    private static class Loader implements CacheLoader<Key, BytesReference> {
+    private static class Loader implements org.opensearch.indices.TieredCacheLoader<Key, BytesReference> {
 
         private final CacheEntity entity;
         private final CheckedSupplier<BytesReference, IOException> loader;
@@ -198,7 +221,7 @@ public boolean isLoaded() {
         @Override
         public BytesReference load(Key key) throws Exception {
             BytesReference value = loader.get();
-            entity.onCached(key, value);
+            // entity.onCached(key, value);
             loaded = true;
             return value;
         }
@@ -212,7 +235,7 @@ interface CacheEntity extends Accountable {
         /**
          * Called after the value was loaded.
          */
-        void onCached(Key key, BytesReference value);
+        void onCached(Key key, BytesReference value, TierType tierType);
 
         /**
          * Returns <code>true</code> iff the resource behind this entity is still open ie.
@@ -229,12 +252,12 @@ interface CacheEntity extends Accountable {
         /**
          * Called each time this entity has a cache hit.
          */
-        void onHit();
+        void onHit(TierType tierType);
 
         /**
          * Called each time this entity has a cache miss.
          */
-        void onMiss();
+        void onMiss(TierType tierType);
 
         /**
          * Called when this entity instance is removed
@@ -344,7 +367,7 @@ synchronized void cleanCache() {
             }
         }
         if (!currentKeysToClean.isEmpty() || !currentFullClean.isEmpty()) {
-            for (Iterator<Key> iterator = cache.keys().iterator(); iterator.hasNext();) {
+            for (Iterator<Key> iterator = tieredCacheHandler.getOnHeapCachingTier().keys().iterator(); iterator.hasNext();) {
                 Key key = iterator.next();
                 if (currentFullClean.contains(key.entity.getCacheIdentity())) {
                     iterator.remove();
@@ -355,15 +378,15 @@ synchronized void cleanCache() {
                 }
             }
         }
-
-        cache.refresh();
+        // TODO
+        // cache.refresh();
     }
 
     /**
      * Returns the current size of the cache
      */
-    int count() {
-        return cache.count();
+    long count() {
+        return tieredCacheHandler.count();
     }
 
     int numRegisteredCloseListeners() { // for testing
diff --git a/server/src/main/java/org/opensearch/indices/OnHeapCachingTier.java b/server/src/main/java/org/opensearch/indices/OnHeapCachingTier.java
new file mode 100644
index 0000000000000..ea8bd79e1d445
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/OnHeapCachingTier.java
@@ -0,0 +1,11 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+public interface OnHeapCachingTier<K, V> extends CachingTier<K, V> {}
diff --git a/server/src/main/java/org/opensearch/indices/OpenSearchOnHeapCache.java b/server/src/main/java/org/opensearch/indices/OpenSearchOnHeapCache.java
new file mode 100644
index 0000000000000..cb250edfd7a24
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/OpenSearchOnHeapCache.java
@@ -0,0 +1,119 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.cache.Cache;
+import org.opensearch.common.cache.CacheBuilder;
+import org.opensearch.common.cache.RemovalListener;
+import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.unit.TimeValue;
+
+import java.util.concurrent.ExecutionException;
+import java.util.function.ToLongBiFunction;
+
+public class OpenSearchOnHeapCache<K, V> implements OnHeapCachingTier<K, V>, RemovalListener<K, V> {
+
+    private final Cache<K, V> cache;
+    private RemovalListener<K, V> removalListener;
+
+    private OpenSearchOnHeapCache(long maxWeightInBytes, ToLongBiFunction<K, V> weigher, TimeValue expireAfterAcess) {
+        CacheBuilder<K, V> cacheBuilder = CacheBuilder.<K, V>builder()
+            .setMaximumWeight(maxWeightInBytes)
+            .weigher(weigher)
+            .removalListener(this);
+        if (expireAfterAcess != null) {
+            cacheBuilder.setExpireAfterAccess(expireAfterAcess);
+        }
+        cache = cacheBuilder.build();
+    }
+
+    @Override
+    public void setRemovalListener(RemovalListener<K, V> removalListener) {
+        this.removalListener = removalListener;
+    }
+
+    @Override
+    public void invalidateAll() {
+
+    }
+
+    @Override
+    public Iterable<K> keys() {
+        return this.cache.keys();
+    }
+
+    @Override
+    public int count() {
+        return cache.count();
+    }
+
+    @Override
+    public TierType getTierType() {
+        return TierType.ON_HEAP;
+    }
+
+    @Override
+    public V get(K key) {
+        return cache.get(key);
+    }
+
+    @Override
+    public void put(K key, V value) {
+        cache.put(key, value);
+    }
+
+    @Override
+    public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws ExecutionException {
+        return cache.computeIfAbsent(key, key1 -> loader.load(key));
+    }
+
+    @Override
+    public void invalidate(K key) {
+        cache.invalidate(key);
+    }
+
+    @Override
+    public V compute(K key, TieredCacheLoader<K, V> loader) throws Exception {
+        return cache.compute(key, key1 -> loader.load(key));
+    }
+
+    @Override
+    public void onRemoval(RemovalNotification<K, V> notification) {
+        removalListener.onRemoval(notification);
+    }
+
+    public static class Builder<K, V> {
+        private long maxWeightInBytes;
+
+        private ToLongBiFunction<K, V> weigher;
+
+        private TimeValue expireAfterAcess;
+
+        public Builder() {}
+
+        public Builder<K, V> setMaximumWeight(long sizeInBytes) {
+            this.maxWeightInBytes = sizeInBytes;
+            return this;
+        }
+
+        public Builder<K, V> setWeigher(ToLongBiFunction<K, V> weigher) {
+            this.weigher = weigher;
+            return this;
+        }
+
+        public Builder<K, V> setExpireAfterAccess(TimeValue expireAfterAcess) {
+            this.expireAfterAcess = expireAfterAcess;
+            return this;
+        }
+
+        public OpenSearchOnHeapCache<K, V> build() {
+            return new OpenSearchOnHeapCache<K, V>(maxWeightInBytes, weigher, expireAfterAcess);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/TierType.java b/server/src/main/java/org/opensearch/indices/TierType.java
new file mode 100644
index 0000000000000..9a286fd26151b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/TierType.java
@@ -0,0 +1,15 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+public enum TierType {
+
+    ON_HEAP,
+    DISK;
+}
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheEventListener.java b/server/src/main/java/org/opensearch/indices/TieredCacheEventListener.java
new file mode 100644
index 0000000000000..084ac5a57e0d3
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheEventListener.java
@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.cache.RemovalNotification;
+
+public interface TieredCacheEventListener<K, V> {
+
+    void onMiss(K key, TierType tierType);
+
+    void onRemoval(RemovalNotification<K, V> notification);
+
+    void onHit(K key, V value, TierType tierType);
+
+    void onCached(K key, V value, TierType tierType);
+}
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
new file mode 100644
index 0000000000000..5fe41f5adce94
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
@@ -0,0 +1,24 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+public interface TieredCacheHandler<K, V> {
+
+    V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception;
+
+    V get(K key);
+
+    void invalidate(K key);
+
+    void invalidateAll();
+
+    long count();
+
+    CachingTier<K, V> getOnHeapCachingTier();
+}
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheLoader.java b/server/src/main/java/org/opensearch/indices/TieredCacheLoader.java
new file mode 100644
index 0000000000000..f6bb1a74e973e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheLoader.java
@@ -0,0 +1,15 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+public interface TieredCacheLoader<K, V> {
+    V load(K key) throws Exception;
+
+    boolean isLoaded();
+}
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
new file mode 100644
index 0000000000000..1047f8a6dc2cc
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -0,0 +1,172 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.cache.RemovalListener;
+import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.RemovalReason;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ *
+ * @param <K>
+ * @param <V>
+ */
+public class TieredCacheSpilloverStrategyHandler<K, V> implements TieredCacheHandler<K, V>, RemovalListener<K, V> {
+
+    private final OnHeapCachingTier<K, V> onHeapCachingTier;
+    private final CachingTier<K, V> diskCachingTier;
+    private final TieredCacheEventListener<K, V> tieredCacheEventListener;
+
+    /**
+     * Maintains caching tiers in order of get calls.
+     */
+    private final List<CachingTier<K, V>> cachingTierList;
+
+    private TieredCacheSpilloverStrategyHandler(
+        OnHeapCachingTier<K, V> onHeapCachingTier,
+        CachingTier<K, V> diskCachingTier,
+        TieredCacheEventListener<K, V> tieredCacheEventListener
+    ) {
+        this.onHeapCachingTier = Objects.requireNonNull(onHeapCachingTier);
+        this.diskCachingTier = Objects.requireNonNull(diskCachingTier);
+        this.tieredCacheEventListener = tieredCacheEventListener;
+        this.cachingTierList = Arrays.asList(onHeapCachingTier, diskCachingTier);
+        setRemovalListeners();
+    }
+
+    @Override
+    public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception {
+        CacheValue<V> cacheValue = getValueFromTierCache().apply(key);
+        if (cacheValue == null) {
+            // Add the value to the onHeap cache. Any items if evicted will be moved to lower tier
+            V value = onHeapCachingTier.compute(key, loader);
+            tieredCacheEventListener.onCached(key, value, TierType.ON_HEAP);
+            return value;
+        } else {
+            tieredCacheEventListener.onHit(key, cacheValue.value, cacheValue.source);
+        }
+        return cacheValue.value;
+    }
+
+    @Override
+    public V get(K key) {
+        CacheValue<V> cacheValue = getValueFromTierCache().apply(key);
+        if (cacheValue == null) {
+            return null;
+        }
+        return cacheValue.value;
+    }
+
+    @Override
+    public void invalidate(K key) {
+        // TODO
+    }
+
+    @Override
+    public void invalidateAll() {
+        for (CachingTier<K, V> cachingTier : cachingTierList) {
+            cachingTier.invalidateAll();
+        }
+    }
+
+    @Override
+    public long count() {
+        long totalCount = 0;
+        for (CachingTier<K, V> cachingTier : cachingTierList) {
+            totalCount += cachingTier.count();
+        }
+        return totalCount;
+    }
+
+    @Override
+    public void onRemoval(RemovalNotification<K, V> notification) {
+        if (RemovalReason.EVICTED.equals(notification.getRemovalReason())) {
+            switch (notification.getTierType()) {
+                case ON_HEAP:
+                    diskCachingTier.put(notification.getKey(), notification.getValue());
+                    break;
+                default:
+                    break;
+            }
+        }
+        tieredCacheEventListener.onRemoval(notification);
+    }
+
+    @Override
+    public CachingTier<K, V> getOnHeapCachingTier() {
+        return this.onHeapCachingTier;
+    }
+
+    private void setRemovalListeners() {
+        for (CachingTier<K, V> cachingTier : cachingTierList) {
+            cachingTier.setRemovalListener(this);
+        }
+    }
+
+    private Function<K, CacheValue<V>> getValueFromTierCache() {
+        return key -> {
+            for (CachingTier<K, V> cachingTier : cachingTierList) {
+                V value = cachingTier.get(key);
+                if (value != null) {
+                    tieredCacheEventListener.onHit(key, value, cachingTier.getTierType());
+                    return new CacheValue<>(value, cachingTier.getTierType());
+                }
+                tieredCacheEventListener.onMiss(key, cachingTier.getTierType());
+            }
+            return null;
+        };
+    }
+
+    public static class CacheValue<V> {
+        V value;
+        TierType source;
+
+        CacheValue(V value, TierType source) {
+            this.value = value;
+            this.source = source;
+        }
+    }
+
+    public static class Builder<K, V> {
+        private OnHeapCachingTier<K, V> onHeapCachingTier;
+        private CachingTier<K, V> diskCachingTier;
+        private TieredCacheEventListener<K, V> tieredCacheEventListener;
+
+        public Builder() {}
+
+        public Builder<K, V> setOnHeapCachingTier(OnHeapCachingTier<K, V> onHeapCachingTier) {
+            this.onHeapCachingTier = onHeapCachingTier;
+            return this;
+        }
+
+        public Builder<K, V> setOnDiskCachingTier(CachingTier<K, V> diskCachingTier) {
+            this.diskCachingTier = diskCachingTier;
+            return this;
+        }
+
+        public Builder<K, V> setTieredCacheEventListener(TieredCacheEventListener<K, V> tieredCacheEventListener) {
+            this.tieredCacheEventListener = tieredCacheEventListener;
+            return this;
+        }
+
+        public TieredCacheSpilloverStrategyHandler<K, V> build() {
+            return new TieredCacheSpilloverStrategyHandler<K, V>(
+                this.onHeapCachingTier,
+                this.diskCachingTier,
+                this.tieredCacheEventListener
+            );
+        }
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java b/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java
index 415844dccb611..2fe1f87f5d828 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java
@@ -321,7 +321,7 @@ public long ramBytesUsed() {
             }
 
             @Override
-            public void onCached(Key key, BytesReference value) {}
+            public void onCached(Key key, BytesReference value, TierType tierType) {}
 
             @Override
             public boolean isOpen() {
@@ -334,10 +334,10 @@ public Object getCacheIdentity() {
             }
 
             @Override
-            public void onHit() {}
+            public void onHit(TierType tierType) {}
 
             @Override
-            public void onMiss() {}
+            public void onMiss(TierType tierType) {}
 
             @Override
             public void onRemoval(RemovalNotification<Key, BytesReference> notification) {}

From 83590e1340e8b093605b4706ad880b333786819c Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 9 Oct 2023 17:01:34 -0700
Subject: [PATCH 007/120] initial commit

---
 .../main/resources/fips_java_bcjsse_11.policy |   6 +
 .../crypto-kms/licenses/slf4j-api-LICENSE.txt |  21 --
 .../licenses/slf4j-api-1.7.36.jar.sha1        |   1 -
 .../licenses/slf4j-api-LICENSE.txt            |  21 --
 .../licenses/slf4j-api-NOTICE.txt             |   0
 .../licenses/slf4j-api-1.7.36.jar.sha1        |   1 -
 .../licenses/slf4j-api-NOTICE.txt             |   0
 .../licenses/slf4j-api-1.7.36.jar.sha1        |   1 -
 .../licenses/slf4j-api-1.7.36.jar.sha1        |   1 -
 .../licenses/slf4j-api-1.7.36.jar.sha1        |   1 -
 .../licenses/slf4j-api-1.7.36.jar.sha1        |   1 -
 server/build.gradle                           |   4 +
 server/licenses/ehcache-3.10.8.jar.sha1       |   1 +
 server/licenses/ehcache-LICENSE.txt           | 201 ++++++++++++++++++
 server/licenses/ehcache-NOTICE.txt            |   5 +
 .../licenses/slf4j-api-1.7.36.jar.sha1        |   0
 .../licenses/slf4j-api-LICENSE.txt            |   4 +-
 .../licenses/slf4j-api-NOTICE.txt             |   0
 .../indices/EhcacheDiskCachingTier.java       | 113 ++++++++++
 .../indices/IndicesRequestCache.java          |   5 +-
 20 files changed, 337 insertions(+), 50 deletions(-)
 delete mode 100644 plugins/crypto-kms/licenses/slf4j-api-LICENSE.txt
 delete mode 100644 plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt
 delete mode 100644 plugins/discovery-ec2/licenses/slf4j-api-NOTICE.txt
 delete mode 100644 plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1
 delete mode 100644 plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt
 delete mode 100644 plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1
 delete mode 100644 plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1
 create mode 100644 server/licenses/ehcache-3.10.8.jar.sha1
 create mode 100644 server/licenses/ehcache-LICENSE.txt
 create mode 100644 server/licenses/ehcache-NOTICE.txt
 rename {plugins/crypto-kms => server}/licenses/slf4j-api-1.7.36.jar.sha1 (100%)
 rename {plugins/identity-shiro => server}/licenses/slf4j-api-LICENSE.txt (95%)
 rename {plugins/crypto-kms => server}/licenses/slf4j-api-NOTICE.txt (100%)
 create mode 100644 server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java

diff --git a/buildSrc/src/main/resources/fips_java_bcjsse_11.policy b/buildSrc/src/main/resources/fips_java_bcjsse_11.policy
index 10193f4eb385d..4c4b9dc670a65 100644
--- a/buildSrc/src/main/resources/fips_java_bcjsse_11.policy
+++ b/buildSrc/src/main/resources/fips_java_bcjsse_11.policy
@@ -26,4 +26,10 @@ grant {
      permission org.bouncycastle.crypto.CryptoServicesPermission "exportSecretKey";
      permission org.bouncycastle.crypto.CryptoServicesPermission "exportPrivateKey";
      permission java.io.FilePermission "${javax.net.ssl.trustStore}", "read";
+     //permission java.lang.RuntimePermission "createClassLoader"; // needed to create an ehcache disk tier cache
+};
+
+grant codeBase "${codebase.ehcache}" {
+  // needed to allow ehcache to create the classloader
+  permission java.security.AllPermission;
 };
diff --git a/plugins/crypto-kms/licenses/slf4j-api-LICENSE.txt b/plugins/crypto-kms/licenses/slf4j-api-LICENSE.txt
deleted file mode 100644
index 2be7689435062..0000000000000
--- a/plugins/crypto-kms/licenses/slf4j-api-LICENSE.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-Copyright (c) 2004-2022 QOS.ch
-All rights reserved.
-
-Permission is hereby granted, free  of charge, to any person obtaining
-a  copy  of this  software  and  associated  documentation files  (the
-"Software"), to  deal in  the Software without  restriction, including
-without limitation  the rights to  use, copy, modify,  merge, publish,
-distribute,  sublicense, and/or sell  copies of  the Software,  and to
-permit persons to whom the Software  is furnished to do so, subject to
-the following conditions:
-
-The  above  copyright  notice  and  this permission  notice  shall  be
-included in all copies or substantial portions of the Software.
-
-THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
-EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
-MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1
deleted file mode 100644
index 77b9917528382..0000000000000
--- a/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6c62681a2f655b49963a5983b8b0950a6120ae14
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt b/plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt
deleted file mode 100644
index 2be7689435062..0000000000000
--- a/plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-Copyright (c) 2004-2022 QOS.ch
-All rights reserved.
-
-Permission is hereby granted, free  of charge, to any person obtaining
-a  copy  of this  software  and  associated  documentation files  (the
-"Software"), to  deal in  the Software without  restriction, including
-without limitation  the rights to  use, copy, modify,  merge, publish,
-distribute,  sublicense, and/or sell  copies of  the Software,  and to
-permit persons to whom the Software  is furnished to do so, subject to
-the following conditions:
-
-The  above  copyright  notice  and  this permission  notice  shall  be
-included in all copies or substantial portions of the Software.
-
-THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
-EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
-MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/slf4j-api-NOTICE.txt b/plugins/discovery-ec2/licenses/slf4j-api-NOTICE.txt
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1
deleted file mode 100644
index 77b9917528382..0000000000000
--- a/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6c62681a2f655b49963a5983b8b0950a6120ae14
\ No newline at end of file
diff --git a/plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt b/plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1
deleted file mode 100644
index 77b9917528382..0000000000000
--- a/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6c62681a2f655b49963a5983b8b0950a6120ae14
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1
deleted file mode 100644
index 77b9917528382..0000000000000
--- a/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6c62681a2f655b49963a5983b8b0950a6120ae14
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1
deleted file mode 100644
index 77b9917528382..0000000000000
--- a/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6c62681a2f655b49963a5983b8b0950a6120ae14
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1
deleted file mode 100644
index 77b9917528382..0000000000000
--- a/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6c62681a2f655b49963a5983b8b0950a6120ae14
\ No newline at end of file
diff --git a/server/build.gradle b/server/build.gradle
index f6db3d53a0dcc..77cc411677ac5 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -158,6 +158,10 @@ dependencies {
   api "com.google.protobuf:protobuf-java:${versions.protobuf}"
   api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}"
 
+  // ehcache
+  api "org.ehcache:ehcache:3.10.8"
+  api "org.slf4j:slf4j-api:1.7.36"
+
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server
     exclude group: 'org.opensearch', module: 'server'
diff --git a/server/licenses/ehcache-3.10.8.jar.sha1 b/server/licenses/ehcache-3.10.8.jar.sha1
new file mode 100644
index 0000000000000..dee07e9238ebf
--- /dev/null
+++ b/server/licenses/ehcache-3.10.8.jar.sha1
@@ -0,0 +1 @@
+f0d50ede46609db78413ca7f4250d348a597b101
\ No newline at end of file
diff --git a/server/licenses/ehcache-LICENSE.txt b/server/licenses/ehcache-LICENSE.txt
new file mode 100644
index 0000000000000..8dada3edaf50d
--- /dev/null
+++ b/server/licenses/ehcache-LICENSE.txt
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/server/licenses/ehcache-NOTICE.txt b/server/licenses/ehcache-NOTICE.txt
new file mode 100644
index 0000000000000..1dbd38242cc98
--- /dev/null
+++ b/server/licenses/ehcache-NOTICE.txt
@@ -0,0 +1,5 @@
+Ehcache V3
+Copyright 2014-2023 Terracotta, Inc.
+
+The product includes software from the Apache Commons Lang project,
+under the Apache License 2.0 (see: org.ehcache.impl.internal.classes.commonslang)
diff --git a/plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1 b/server/licenses/slf4j-api-1.7.36.jar.sha1
similarity index 100%
rename from plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1
rename to server/licenses/slf4j-api-1.7.36.jar.sha1
diff --git a/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt b/server/licenses/slf4j-api-LICENSE.txt
similarity index 95%
rename from plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt
rename to server/licenses/slf4j-api-LICENSE.txt
index 8fda22f4d72f6..a51675a21c10f 100644
--- a/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt
+++ b/server/licenses/slf4j-api-LICENSE.txt
@@ -1,4 +1,4 @@
-Copyright (c) 2004-2014 QOS.ch
+Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland)
 All rights reserved.
 
 Permission is hereby granted, free  of charge, to any person obtaining
@@ -19,3 +19,5 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
diff --git a/plugins/crypto-kms/licenses/slf4j-api-NOTICE.txt b/server/licenses/slf4j-api-NOTICE.txt
similarity index 100%
rename from plugins/crypto-kms/licenses/slf4j-api-NOTICE.txt
rename to server/licenses/slf4j-api-NOTICE.txt
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
new file mode 100644
index 0000000000000..960b5f76f9ac7
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -0,0 +1,113 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.ehcache.config.builders.CacheConfigurationBuilder;
+import org.ehcache.config.builders.CacheManagerBuilder;
+import org.ehcache.config.builders.ResourcePoolsBuilder;
+import org.ehcache.config.units.MemoryUnit;
+import org.opensearch.common.cache.RemovalListener;
+import org.ehcache.Cache;
+import org.ehcache.CacheManager;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+
+public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V> {
+
+    private CacheManager cacheManager;
+    private Cache<K, V> cache;
+    private final Class<K> keyType; // I think these are needed to pass to newCacheConfigurationBuilder
+    private final Class<V> valueType;
+    //private final Path DISK_CACHE_FP = Paths.get("disk_cache_tier"); // ?
+    private final String DISK_CACHE_FP = "disk_cache_tier";
+    // private RBMIntKeyLookupStore keystore;
+    // private CacheTierPolicy[] policies;
+    // private IndicesRequestCacheDiskTierPolicy policy;
+    private RemovalListener<K, V> removalListener = notification -> {}; // based on on-heap
+
+    public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInBytes, Class<K> keyType, Class<V> valueType) {
+        this.keyType = keyType;
+        this.valueType = valueType;
+        String cacheAlias = "diskTier";
+        /*try {
+            Path file = Files.createFile(DISK_CACHE_FP);
+        } catch (Exception ignored) {}*/
+        this.cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
+            //.with(CacheManagerBuilder.persistence(new File(DISK_CACHE_FP)))
+            .with(CacheManagerBuilder.persistence(DISK_CACHE_FP))
+            .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
+                keyType, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, true)
+            )).build(true);
+        /*this.cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
+            .with(CacheManagerBuilder.persistence(new File(DISK_CACHE_FP)))
+            .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
+                IndicesRequestCache.Key.class, BytesReference.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, true)
+            )).build(true);*/
+        this.cache = cacheManager.getCache(cacheAlias, keyType, valueType);
+        // this.keystore = new RBMIntKeyLookupStore((int) Math.pow(2, 28), maxKeystoreWeightInBytes);
+        // this.policies = new CacheTierPolicy[]{ new IndicesRequestCacheTookTimePolicy(settings, clusterSettings) };
+        // this.policy = new IndicesRequestCacheDiskTierPolicy(this.policies, true);
+    }
+
+    @Override
+    public V get(K key) {
+        // keystore check goes here
+        // if (keystore.contains(key.hashCode())
+        // do we need to do the future stuff?
+        return cache.get(key); // do we want to somehow keep track of how long each cache.get takes?
+        //return null;
+    }
+
+    @Override
+    public void put(K key, V value) {
+        // CheckDataResult policyResult = policy.checkData(value)
+        // if (policyResult.isAccepted()) {
+        cache.put(key, value);
+        // do we need to check for replacement, as onheap does?
+        // else { do something with policyResult.deniedReason()? }
+    }
+
+    @Override
+    public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception {
+        return null;
+    }
+
+    @Override
+    public void invalidate(K key) {}
+
+    @Override
+    public V compute(K key, TieredCacheLoader<K, V> loader) throws Exception {
+        return null;
+    }
+
+    @Override
+    public void setRemovalListener(RemovalListener<K, V> removalListener) {}
+
+    @Override
+    public void invalidateAll() {}
+
+    @Override
+    public Iterable<K> keys() {
+        return Collections::emptyIterator;
+    }
+
+    @Override
+    public int count() {
+        return 0;
+    }
+
+    @Override
+    public TierType getTierType() {
+        return TierType.DISK;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index ab80e63a4abce..c43e09992be20 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -124,9 +124,12 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             (k, v) -> k.ramBytesUsed() + v.ramBytesUsed()
         ).setMaximumWeight(sizeInBytes).setExpireAfterAccess(expire).build();
 
+        int diskTierWeight = 100 * 1048576; // 100 MB, for testing only
+        EhcacheDiskCachingTier<Key, BytesReference> diskCachingTier;
+        diskCachingTier = new EhcacheDiskCachingTier<>(diskTierWeight, 0, Key.class, BytesReference.class);
         tieredCacheHandler = new TieredCacheSpilloverStrategyHandler.Builder<Key, BytesReference>().setOnHeapCachingTier(
             openSearchOnHeapCache
-        ).setOnDiskCachingTier(new DummyDiskCachingTier<>()).setTieredCacheEventListener(this).build();
+        ).setOnDiskCachingTier(diskCachingTier).setTieredCacheEventListener(this).build();
     }
 
     @Override

From aa0a5d7b555209de3384e309b13a114ac157e462 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 10 Oct 2023 10:24:28 -0700
Subject: [PATCH 008/120] fixed permissions

---
 buildSrc/src/main/resources/fips_java_bcjsse_11.policy |  6 ------
 server/disk_cache_tier/.lock                           |  0
 .../org/opensearch/indices/EhcacheDiskCachingTier.java | 10 ----------
 .../resources/org/opensearch/bootstrap/security.policy |  8 ++++++++
 4 files changed, 8 insertions(+), 16 deletions(-)
 create mode 100644 server/disk_cache_tier/.lock

diff --git a/buildSrc/src/main/resources/fips_java_bcjsse_11.policy b/buildSrc/src/main/resources/fips_java_bcjsse_11.policy
index 4c4b9dc670a65..10193f4eb385d 100644
--- a/buildSrc/src/main/resources/fips_java_bcjsse_11.policy
+++ b/buildSrc/src/main/resources/fips_java_bcjsse_11.policy
@@ -26,10 +26,4 @@ grant {
      permission org.bouncycastle.crypto.CryptoServicesPermission "exportSecretKey";
      permission org.bouncycastle.crypto.CryptoServicesPermission "exportPrivateKey";
      permission java.io.FilePermission "${javax.net.ssl.trustStore}", "read";
-     //permission java.lang.RuntimePermission "createClassLoader"; // needed to create an ehcache disk tier cache
-};
-
-grant codeBase "${codebase.ehcache}" {
-  // needed to allow ehcache to create the classloader
-  permission java.security.AllPermission;
 };
diff --git a/server/disk_cache_tier/.lock b/server/disk_cache_tier/.lock
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 960b5f76f9ac7..fdbc38fe6a716 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -28,7 +28,6 @@ public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V> {
     private Cache<K, V> cache;
     private final Class<K> keyType; // I think these are needed to pass to newCacheConfigurationBuilder
     private final Class<V> valueType;
-    //private final Path DISK_CACHE_FP = Paths.get("disk_cache_tier"); // ?
     private final String DISK_CACHE_FP = "disk_cache_tier";
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
@@ -39,20 +38,11 @@ public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInByt
         this.keyType = keyType;
         this.valueType = valueType;
         String cacheAlias = "diskTier";
-        /*try {
-            Path file = Files.createFile(DISK_CACHE_FP);
-        } catch (Exception ignored) {}*/
         this.cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
-            //.with(CacheManagerBuilder.persistence(new File(DISK_CACHE_FP)))
             .with(CacheManagerBuilder.persistence(DISK_CACHE_FP))
             .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
                 keyType, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, true)
             )).build(true);
-        /*this.cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
-            .with(CacheManagerBuilder.persistence(new File(DISK_CACHE_FP)))
-            .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                IndicesRequestCache.Key.class, BytesReference.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, true)
-            )).build(true);*/
         this.cache = cacheManager.getCache(cacheAlias, keyType, valueType);
         // this.keystore = new RBMIntKeyLookupStore((int) Math.pow(2, 28), maxKeystoreWeightInBytes);
         // this.policies = new CacheTierPolicy[]{ new IndicesRequestCacheTookTimePolicy(settings, clusterSettings) };
diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy
index 77cd0ab05278e..20e5671d582a6 100644
--- a/server/src/main/resources/org/opensearch/bootstrap/security.policy
+++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy
@@ -188,4 +188,12 @@ grant {
   permission java.io.FilePermission "/sys/fs/cgroup/memory", "read";
   permission java.io.FilePermission "/sys/fs/cgroup/memory/-", "read";
 
+  // ehcache
+  permission java.lang.RuntimePermission "createClassLoader";
+  permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
+  permission java.lang.RuntimePermission "getenv.*";
+  permission java.io.FilePermission "disk_cache_tier", "read"; // change this to wherever we will put disk tier folder
+  permission java.io.FilePermission "disk_cache_tier", "write";
+  permission java.io.FilePermission "disk_cache_tier/-", "read";
+  permission java.io.FilePermission "disk_cache_tier/-", "write";
 };

From 41dc035643f0962a3a8e58338ebba3dd498ae7a7 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 10 Oct 2023 16:26:52 -0700
Subject: [PATCH 009/120] Set up event listener to create RemovalNotification
 objects, also misc functions

---
 .../core/common/bytes/BytesReference.java     |   3 +-
 .../ehcache-disk-store.data                   | Bin 0 -> 16384 bytes
 .../ehcache-disk-store.meta                   |   4 +
 .../indices/EhcacheDiskCachingTier.java       | 121 +++++++++++++++---
 .../indices/IndicesRequestCache.java          |   6 +-
 5 files changed, 113 insertions(+), 21 deletions(-)
 create mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data
 create mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta

diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
index 9d24d3653397b..60613d4e2a997 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
@@ -45,6 +45,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 
@@ -54,7 +55,7 @@
  * @opensearch.api
  */
 @PublicApi(since = "1.0.0")
-public interface BytesReference extends Comparable<BytesReference>, ToXContentFragment {
+public interface BytesReference extends Comparable<BytesReference>, ToXContentFragment, Serializable { // another lie!
 
     /**
      * Convert an {@link XContentBuilder} into a BytesReference. This method closes the builder,
diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data
new file mode 100644
index 0000000000000000000000000000000000000000..294f4016d05bdd696670c4840f1f36a71f9239de
GIT binary patch
literal 16384
zcmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM
q7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjMyaxtA00031

literal 0
HcmV?d00001

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
new file mode 100644
index 0000000000000..2d3e90d195f10
--- /dev/null
+++ b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
@@ -0,0 +1,4 @@
+#Key and value types
+#Tue Oct 10 19:13:45 EDT 2023
+valueType=org.opensearch.core.common.bytes.BytesReference
+keyType=org.opensearch.indices.IndicesRequestCache$Key
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index fdbc38fe6a716..42445bc3ea6ef 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -9,41 +9,65 @@
 package org.opensearch.indices;
 
 import org.ehcache.config.builders.CacheConfigurationBuilder;
+import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
 import org.ehcache.config.builders.CacheManagerBuilder;
 import org.ehcache.config.builders.ResourcePoolsBuilder;
 import org.ehcache.config.units.MemoryUnit;
+import org.ehcache.core.internal.statistics.DefaultStatisticsService;
+import org.ehcache.core.spi.service.StatisticsService;
+import org.ehcache.core.statistics.CacheStatistics;
+import org.ehcache.event.CacheEvent;
+import org.ehcache.event.CacheEventListener;
+import org.ehcache.event.EventType;
 import org.opensearch.common.cache.RemovalListener;
 import org.ehcache.Cache;
 import org.ehcache.CacheManager;
+import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.RemovalReason;
 
-import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.Collections;
 
-public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V> {
+public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalListener<K, V> {
 
     private CacheManager cacheManager;
     private Cache<K, V> cache;
+
     private final Class<K> keyType; // I think these are needed to pass to newCacheConfigurationBuilder
     private final Class<V> valueType;
     private final String DISK_CACHE_FP = "disk_cache_tier";
+    private RemovalListener<K, V> removalListener;
+    private final StatisticsService statsService;
+    private final CacheStatistics cacheStats;
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
     // private IndicesRequestCacheDiskTierPolicy policy;
-    private RemovalListener<K, V> removalListener = notification -> {}; // based on on-heap
+    //private RemovalListener<K, V> removalListener = notification -> {}; // based on on-heap
 
     public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInBytes, Class<K> keyType, Class<V> valueType) {
         this.keyType = keyType;
         this.valueType = valueType;
         String cacheAlias = "diskTier";
+        this.statsService = new DefaultStatisticsService();
+
+        // our EhcacheEventListener should receive events every time an entry is removed for some reason, but not when it's created
+        CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
+            .newEventListenerConfiguration(new EhcacheEventListener(this),
+                EventType.EVICTED,
+                EventType.EXPIRED,
+                EventType.REMOVED,
+                EventType.UPDATED)
+            .ordered().asynchronous(); // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
+
         this.cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
+            .using(statsService)
             .with(CacheManagerBuilder.persistence(DISK_CACHE_FP))
             .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                keyType, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, true)
-            )).build(true);
+                keyType, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, true))
+                .withService(listenerConfig)
+            ).build(true);
         this.cache = cacheManager.getCache(cacheAlias, keyType, valueType);
+        this.cacheStats = statsService.getCacheStatistics(cacheAlias);
+
         // this.keystore = new RBMIntKeyLookupStore((int) Math.pow(2, 28), maxKeystoreWeightInBytes);
         // this.policies = new CacheTierPolicy[]{ new IndicesRequestCacheTookTimePolicy(settings, clusterSettings) };
         // this.policy = new IndicesRequestCacheDiskTierPolicy(this.policies, true);
@@ -51,53 +75,112 @@ public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInByt
 
     @Override
     public V get(K key) {
-        // keystore check goes here
-        // if (keystore.contains(key.hashCode())
-        // do we need to do the future stuff?
+        // do we need to do the future stuff? I don't think so?
+
+        // if (keystore.contains(key.hashCode()) {
         return cache.get(key); // do we want to somehow keep track of how long each cache.get takes?
-        //return null;
+        // }
+        // return null;
+
     }
 
     @Override
     public void put(K key, V value) {
+        // No need to get old value, this is handled by EhcacheEventListener.
+
         // CheckDataResult policyResult = policy.checkData(value)
         // if (policyResult.isAccepted()) {
         cache.put(key, value);
-        // do we need to check for replacement, as onheap does?
         // else { do something with policyResult.deniedReason()? }
+        // }
     }
 
     @Override
     public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception {
-        return null;
+        return null; // should not need to fill out, Cache.computeIfAbsent is always used
     }
 
     @Override
-    public void invalidate(K key) {}
+    public void invalidate(K key) {
+        // keep keystore check to avoid unneeded disk seek
+        // RemovalNotification is handled by EhcacheEventListener
+
+        // if (keystore.contains(key.hashCode()) {
+        cache.remove(key);
+        // }
+    }
 
     @Override
     public V compute(K key, TieredCacheLoader<K, V> loader) throws Exception {
-        return null;
+        return null; // should not need to fill out, Cache.compute is always used
     }
 
     @Override
-    public void setRemovalListener(RemovalListener<K, V> removalListener) {}
+    public void setRemovalListener(RemovalListener<K, V> removalListener) {
+        this.removalListener = removalListener; // this is passed the spillover strategy, same as on-heap
+    }
 
     @Override
-    public void invalidateAll() {}
+    public void invalidateAll() {
+        // can we just wipe the cache and start over? Or do we need to create a bunch of RemovalNotifications?
+    }
 
     @Override
     public Iterable<K> keys() {
+        // ehcache doesn't provide a method like this, because it might be a huge number of keys that consume all
+        // the memory. Do we want this method for disk tier?
         return Collections::emptyIterator;
     }
 
     @Override
     public int count() {
-        return 0;
+        // this might be an expensive disk-seek call. Might be better to keep track ourselves?
+        return (int) cacheStats.getTierStatistics().get("Disk").getMappings();
     }
 
     @Override
     public TierType getTierType() {
         return TierType.DISK;
     }
+
+    @Override
+    public void onRemoval(RemovalNotification<K, V> notification) {
+        removalListener.onRemoval(notification);
+    }
+
+    // See https://stackoverflow.com/questions/45827753/listenerobject-not-found-in-imports-for-ehcache-3 for API reference
+    // This class is used to get the old value from mutating calls to the cache and use those to create a RemovalNotification
+    private class EhcacheEventListener implements CacheEventListener<Object, Object> {
+        private RemovalListener<K, V> removalListener;
+        EhcacheEventListener(RemovalListener<K, V> removalListener) {
+            this.removalListener = removalListener;
+        }
+        @Override
+        public void onEvent(CacheEvent<?, ?> event) {
+            // send a RemovalNotification
+            K key = (K) event.getKey(); // I think these casts should be ok?
+            V oldValue = (V) event.getOldValue();
+            V newValue = (V) event.getNewValue();
+            EventType eventType = event.getType();
+            RemovalReason reason;
+            switch (eventType) {
+                case EVICTED:
+                    reason = RemovalReason.EVICTED; // why is there both RemovalReason.EVICTED and RemovalReason.CAPACITY?
+                    break;
+                case EXPIRED:
+                    reason = RemovalReason.INVALIDATED; // not sure about this one
+                    break;
+                case REMOVED:
+                    reason = RemovalReason.INVALIDATED; // we use cache.remove() to invalidate keys, but this might overlap with RemovalReason.EXPLICIT?
+                    break;
+                case UPDATED:
+                    reason = RemovalReason.REPLACED;
+                    break;
+                default:
+                    reason = null;
+            }
+            // we don't subscribe to CREATED type, which is the only other option
+            removalListener.onRemoval(new RemovalNotification<K, V>(key, oldValue, reason));
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index c43e09992be20..d55c188b8a418 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -46,11 +46,13 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.ConcurrentCollections;
+import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.unit.ByteSizeValue;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -125,6 +127,8 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         ).setMaximumWeight(sizeInBytes).setExpireAfterAccess(expire).build();
 
         int diskTierWeight = 100 * 1048576; // 100 MB, for testing only
+
+        // changed to Integer for testing of bulk writes
         EhcacheDiskCachingTier<Key, BytesReference> diskCachingTier;
         diskCachingTier = new EhcacheDiskCachingTier<>(diskTierWeight, 0, Key.class, BytesReference.class);
         tieredCacheHandler = new TieredCacheSpilloverStrategyHandler.Builder<Key, BytesReference>().setOnHeapCachingTier(
@@ -273,7 +277,7 @@ interface CacheEntity extends Accountable {
      *
      * @opensearch.internal
      */
-    static class Key implements Accountable {
+    static class Key implements Accountable, Serializable { // lies! it doesnt implement serializable in a functioning way, but for testing
         private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Key.class);
 
         public final CacheEntity entity; // use as identity equality

From 4ba49afbca3f26a6b42a297ea2a8094841dee7ed Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 11 Oct 2023 10:54:57 -0700
Subject: [PATCH 010/120] Added misc getter methods

---
 .../indices/EhcacheDiskCachingTier.java       | 62 ++++++++++++++++---
 1 file changed, 53 insertions(+), 9 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 42445bc3ea6ef..3f9057c436be0 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -16,9 +16,11 @@
 import org.ehcache.core.internal.statistics.DefaultStatisticsService;
 import org.ehcache.core.spi.service.StatisticsService;
 import org.ehcache.core.statistics.CacheStatistics;
+import org.ehcache.core.statistics.TierStatistics;
 import org.ehcache.event.CacheEvent;
 import org.ehcache.event.CacheEventListener;
 import org.ehcache.event.EventType;
+import org.opensearch.common.ExponentiallyWeightedMovingAverage;
 import org.opensearch.common.cache.RemovalListener;
 import org.ehcache.Cache;
 import org.ehcache.CacheManager;
@@ -26,18 +28,21 @@
 import org.opensearch.common.cache.RemovalReason;
 
 import java.util.Collections;
+import java.util.Map;
 
 public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalListener<K, V> {
 
     private CacheManager cacheManager;
-    private Cache<K, V> cache;
+    private final Cache<K, V> cache;
 
     private final Class<K> keyType; // I think these are needed to pass to newCacheConfigurationBuilder
     private final Class<V> valueType;
-    private final String DISK_CACHE_FP = "disk_cache_tier";
+    private final String DISK_CACHE_FP = "disk_cache_tier"; // this should probably be defined somewhere else since we need to change security.policy based on its value
     private RemovalListener<K, V> removalListener;
     private final StatisticsService statsService;
     private final CacheStatistics cacheStats;
+    private ExponentiallyWeightedMovingAverage getTimeMillisEWMA;
+    private static final double GET_TIME_EWMA_ALPHA  = 0.3; // This is the value used elsewhere in the code
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
     // private IndicesRequestCacheDiskTierPolicy policy;
@@ -67,6 +72,9 @@ public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInByt
             ).build(true);
         this.cache = cacheManager.getCache(cacheAlias, keyType, valueType);
         this.cacheStats = statsService.getCacheStatistics(cacheAlias);
+        this.getTimeMillisEWMA = new ExponentiallyWeightedMovingAverage(GET_TIME_EWMA_ALPHA, 10);
+
+        // try and feed it an OpenSearch threadpool rather than its default ExecutionService?
 
         // this.keystore = new RBMIntKeyLookupStore((int) Math.pow(2, 28), maxKeystoreWeightInBytes);
         // this.policies = new CacheTierPolicy[]{ new IndicesRequestCacheTookTimePolicy(settings, clusterSettings) };
@@ -78,7 +86,11 @@ public V get(K key) {
         // do we need to do the future stuff? I don't think so?
 
         // if (keystore.contains(key.hashCode()) {
-        return cache.get(key); // do we want to somehow keep track of how long each cache.get takes?
+        long now = System.nanoTime();
+        V value = cache.get(key);
+        double tookTimeMillis = ((double) (System.nanoTime() - now)) / 1000000;
+        getTimeMillisEWMA.addValue(tookTimeMillis);
+        return value;
         // }
         // return null;
 
@@ -91,6 +103,7 @@ public void put(K key, V value) {
         // CheckDataResult policyResult = policy.checkData(value)
         // if (policyResult.isAccepted()) {
         cache.put(key, value);
+        // keystore.add(key.hashCode());
         // else { do something with policyResult.deniedReason()? }
         // }
     }
@@ -107,6 +120,7 @@ public void invalidate(K key) {
 
         // if (keystore.contains(key.hashCode()) {
         cache.remove(key);
+        // keystore.remove(key.hashCode());
         // }
     }
 
@@ -135,7 +149,7 @@ public Iterable<K> keys() {
     @Override
     public int count() {
         // this might be an expensive disk-seek call. Might be better to keep track ourselves?
-        return (int) cacheStats.getTierStatistics().get("Disk").getMappings();
+        return (int) getTierStats().getMappings();
     }
 
     @Override
@@ -148,9 +162,40 @@ public void onRemoval(RemovalNotification<K, V> notification) {
         removalListener.onRemoval(notification);
     }
 
+    public double getTimeMillisEWMA() {
+        return getTimeMillisEWMA.getAverage();
+    }
+
+    private TierStatistics getTierStats() {
+        return cacheStats.getTierStatistics().get("Disk");
+    }
+
+    public long getHits() {
+        return getTierStats().getHits();
+    }
+
+    public long getMisses() {
+        return getTierStats().getMisses();
+    }
+
+    public long getWeightBytes() {
+        return getTierStats().getOccupiedByteSize();
+    }
+
+    public long getEvictions() {
+        return getTierStats().getEvictions();
+    }
+
+    public double getHitRatio() {
+        TierStatistics ts = getTierStats();
+        long hits = ts.getHits();
+        return hits / (hits + ts.getMisses());
+    }
+
     // See https://stackoverflow.com/questions/45827753/listenerobject-not-found-in-imports-for-ehcache-3 for API reference
-    // This class is used to get the old value from mutating calls to the cache and use those to create a RemovalNotification
-    private class EhcacheEventListener implements CacheEventListener<Object, Object> {
+    // it's not actually documented by ehcache :(
+    // This class is used to get the old value from mutating calls to the cache, and it uses those to create a RemovalNotification
+    private class EhcacheEventListener implements CacheEventListener<Object, Object> { // try making these specific, but i dont think itll work
         private RemovalListener<K, V> removalListener;
         EhcacheEventListener(RemovalListener<K, V> removalListener) {
             this.removalListener = removalListener;
@@ -168,10 +213,9 @@ public void onEvent(CacheEvent<?, ?> event) {
                     reason = RemovalReason.EVICTED; // why is there both RemovalReason.EVICTED and RemovalReason.CAPACITY?
                     break;
                 case EXPIRED:
-                    reason = RemovalReason.INVALIDATED; // not sure about this one
-                    break;
                 case REMOVED:
-                    reason = RemovalReason.INVALIDATED; // we use cache.remove() to invalidate keys, but this might overlap with RemovalReason.EXPLICIT?
+                    reason = RemovalReason.INVALIDATED;
+                    // this is probably fine for EXPIRED. We use cache.remove() to invalidate keys, but this might overlap with RemovalReason.EXPLICIT?
                     break;
                 case UPDATED:
                     reason = RemovalReason.REPLACED;

From dcb9fad0cc76162d61045d4ef87638f33d8a7698 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 11 Oct 2023 14:17:42 -0700
Subject: [PATCH 011/120] thread pool stuff

---
 .../offheap-disk-store/ehcache-disk-store.meta   |  2 +-
 .../indices/EhcacheDiskCachingTier.java          | 16 ++++++++++++----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
index 2d3e90d195f10..3165bdaf28d86 100644
--- a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
+++ b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
@@ -1,4 +1,4 @@
 #Key and value types
-#Tue Oct 10 19:13:45 EDT 2023
+#Wed Oct 11 23:48:40 GMT+03:00 2023
 valueType=org.opensearch.core.common.bytes.BytesReference
 keyType=org.opensearch.indices.IndicesRequestCache$Key
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 3f9057c436be0..b28c159ffda17 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -11,6 +11,7 @@
 import org.ehcache.config.builders.CacheConfigurationBuilder;
 import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
 import org.ehcache.config.builders.CacheManagerBuilder;
+import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
 import org.ehcache.config.builders.ResourcePoolsBuilder;
 import org.ehcache.config.units.MemoryUnit;
 import org.ehcache.core.internal.statistics.DefaultStatisticsService;
@@ -20,6 +21,7 @@
 import org.ehcache.event.CacheEvent;
 import org.ehcache.event.CacheEventListener;
 import org.ehcache.event.EventType;
+import org.ehcache.impl.config.executor.PooledExecutionServiceConfiguration;
 import org.opensearch.common.ExponentiallyWeightedMovingAverage;
 import org.opensearch.common.cache.RemovalListener;
 import org.ehcache.Cache;
@@ -28,7 +30,6 @@
 import org.opensearch.common.cache.RemovalReason;
 
 import java.util.Collections;
-import java.util.Map;
 
 public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalListener<K, V> {
 
@@ -39,10 +40,11 @@ public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalL
     private final Class<V> valueType;
     private final String DISK_CACHE_FP = "disk_cache_tier"; // this should probably be defined somewhere else since we need to change security.policy based on its value
     private RemovalListener<K, V> removalListener;
-    private final StatisticsService statsService;
     private final CacheStatistics cacheStats;
     private ExponentiallyWeightedMovingAverage getTimeMillisEWMA;
     private static final double GET_TIME_EWMA_ALPHA  = 0.3; // This is the value used elsewhere in the code
+    private static final int MIN_WRITE_THREADS = 0;
+    private static final int MAX_WRITE_THREADS = 4; // Max number of threads for the PooledExecutionService which handles writes
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
     // private IndicesRequestCacheDiskTierPolicy policy;
@@ -52,7 +54,7 @@ public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInByt
         this.keyType = keyType;
         this.valueType = valueType;
         String cacheAlias = "diskTier";
-        this.statsService = new DefaultStatisticsService();
+        StatisticsService statsService = new DefaultStatisticsService();
 
         // our EhcacheEventListener should receive events every time an entry is removed for some reason, but not when it's created
         CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
@@ -63,12 +65,18 @@ public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInByt
                 EventType.UPDATED)
             .ordered().asynchronous(); // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
 
+        // test PooledExecutionService so we know what we might need to replicate
+        PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
+            .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
+            .build();
+
         this.cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
             .using(statsService)
+            .using(threadConfig)
             .with(CacheManagerBuilder.persistence(DISK_CACHE_FP))
             .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
                 keyType, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, true))
-                .withService(listenerConfig)
+                .withService(listenerConfig) // stackoverflow shows .add(), but IDE says this is deprecated. idk
             ).build(true);
         this.cache = cacheManager.getCache(cacheAlias, keyType, valueType);
         this.cacheStats = statsService.getCacheStatistics(cacheAlias);

From b912527b129f0bc48571c87e2686783fb04299e0 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 11 Oct 2023 15:13:17 -0700
Subject: [PATCH 012/120] cleanup

---
 ...CompactJavaSerializer-ObjectStreamClassIndex.bin | Bin 0 -> 4 bytes
 .../opensearch/indices/EhcacheDiskCachingTier.java  |   6 ++----
 2 files changed, 2 insertions(+), 4 deletions(-)
 create mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin
new file mode 100644
index 0000000000000000000000000000000000000000..711006c3d3b5c6d50049e3f48311f3dbe372803d
GIT binary patch
literal 4
LcmZ4UmVp%j1%Lsc

literal 0
HcmV?d00001

diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index b28c159ffda17..2e69fd372c7c6 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -42,13 +42,12 @@ public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalL
     private RemovalListener<K, V> removalListener;
     private final CacheStatistics cacheStats;
     private ExponentiallyWeightedMovingAverage getTimeMillisEWMA;
-    private static final double GET_TIME_EWMA_ALPHA  = 0.3; // This is the value used elsewhere in the code
+    private static final double GET_TIME_EWMA_ALPHA  = 0.3; // This is the value used elsewhere in OpenSearch
     private static final int MIN_WRITE_THREADS = 0;
     private static final int MAX_WRITE_THREADS = 4; // Max number of threads for the PooledExecutionService which handles writes
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
     // private IndicesRequestCacheDiskTierPolicy policy;
-    //private RemovalListener<K, V> removalListener = notification -> {}; // based on on-heap
 
     public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInBytes, Class<K> keyType, Class<V> valueType) {
         this.keyType = keyType;
@@ -56,7 +55,7 @@ public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInByt
         String cacheAlias = "diskTier";
         StatisticsService statsService = new DefaultStatisticsService();
 
-        // our EhcacheEventListener should receive events every time an entry is removed for some reason, but not when it's created
+        // our EhcacheEventListener should receive events every time an entry is removed, but not when it's created
         CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
             .newEventListenerConfiguration(new EhcacheEventListener(this),
                 EventType.EVICTED,
@@ -65,7 +64,6 @@ public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInByt
                 EventType.UPDATED)
             .ordered().asynchronous(); // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
 
-        // test PooledExecutionService so we know what we might need to replicate
         PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
             .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
             .build();

From 44d5e6105ff92bf97481c9612d6429d7af3cdab9 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 11 Oct 2023 15:23:53 -0700
Subject: [PATCH 013/120] Removed partial cache files from disk

---
 server/disk_cache_tier/.lock                    |   0
 .../offheap-disk-store/ehcache-disk-store.data  | Bin 16384 -> 0 bytes
 .../offheap-disk-store/ehcache-disk-store.meta  |   4 ----
 ...actJavaSerializer-ObjectStreamClassIndex.bin | Bin 4 -> 0 bytes
 4 files changed, 4 deletions(-)
 delete mode 100644 server/disk_cache_tier/.lock
 delete mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data
 delete mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
 delete mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin

diff --git a/server/disk_cache_tier/.lock b/server/disk_cache_tier/.lock
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data
deleted file mode 100644
index 294f4016d05bdd696670c4840f1f36a71f9239de..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 16384
zcmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM
q7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjMyaxtA00031

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
deleted file mode 100644
index 3165bdaf28d86..0000000000000
--- a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
+++ /dev/null
@@ -1,4 +0,0 @@
-#Key and value types
-#Wed Oct 11 23:48:40 GMT+03:00 2023
-valueType=org.opensearch.core.common.bytes.BytesReference
-keyType=org.opensearch.indices.IndicesRequestCache$Key
diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin
deleted file mode 100644
index 711006c3d3b5c6d50049e3f48311f3dbe372803d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 4
LcmZ4UmVp%j1%Lsc


From 43e2d1deb4596cf798c4dfed52a2ea5147702f95 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 11 Oct 2023 16:14:50 -0700
Subject: [PATCH 014/120] fixed double counting of on heap hit count

---
 .../opensearch/indices/TieredCacheSpilloverStrategyHandler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index 1047f8a6dc2cc..b1b68bbf93f79 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -54,7 +54,7 @@ public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception
             tieredCacheEventListener.onCached(key, value, TierType.ON_HEAP);
             return value;
         } else {
-            tieredCacheEventListener.onHit(key, cacheValue.value, cacheValue.source);
+            //tieredCacheEventListener.onHit(key, cacheValue.value, cacheValue.source); // this double counts, see line 122
         }
         return cacheValue.value;
     }

From aa2d28d8fef2a02bcfae81cf3aad80d29e818381 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 11 Oct 2023 16:38:03 -0700
Subject: [PATCH 015/120] first attempt at fixing cache closing issue in tests

---
 server/disk_cache_tier/.lock                    |   0
 ...actJavaSerializer-ObjectStreamClassIndex.bin | Bin 0 -> 1367 bytes
 .../offheap-disk-store/ehcache-disk-store.data  | Bin 0 -> 16384 bytes
 .../offheap-disk-store/ehcache-disk-store.index | Bin 0 -> 473 bytes
 .../offheap-disk-store/ehcache-disk-store.meta  |   4 ++++
 ...actJavaSerializer-ObjectStreamClassIndex.bin | Bin 0 -> 1367 bytes
 .../indices/EhcacheDiskCachingTier.java         |   7 +++++++
 .../opensearch/indices/IndicesRequestCache.java |   4 ++++
 .../opensearch/indices/TieredCacheHandler.java  |   2 ++
 .../TieredCacheSpilloverStrategyHandler.java    |  11 ++++++++---
 .../org/opensearch/bootstrap/security.policy    |   2 ++
 .../indices/IndicesRequestCacheTests.java       |   6 ++++++
 12 files changed, 33 insertions(+), 3 deletions(-)
 create mode 100644 server/disk_cache_tier/.lock
 create mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/key-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin
 create mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data
 create mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.index
 create mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
 create mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin

diff --git a/server/disk_cache_tier/.lock b/server/disk_cache_tier/.lock
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/key-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/key-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin
new file mode 100644
index 0000000000000000000000000000000000000000..ef98475e283358e20439f5c5846b5c71b003ab05
GIT binary patch
literal 1367
zcmeHG&ubGw6n;&tS}RyQi(Ud6NO!?%5M!-15*pcB%vKP4nrvRWlkU!}Gqakic=4b|
z!CnNtc=1=os~$v;g8mKd!Gj0!fADRxX;WHD@N5oCX4&t@`@VUrzhT%wp)A9QdFC>^
z#avXJ1^f=G@ws{nJ>wdMEG!31>#AsATPr9@wP>O5G0#Vfhf!cfs5I9Gg@;y!2Y8Wb
zY`BIQtYIWIH&QL<=Hn>9kuzV0K8>uz8OT&&m?w-5?Db{PKxH8B5Vf6B+dATQAG6&d
zoaua8YDYjnGBdHwj7~>jhD%}1D<(8YL4%N;F8bx#gSXGlmew;c<UlrLk%64kqUr6}
z@WA$@@M5J1W0yDn&a#L^NwBz0kPCxKu)x{`TARQbrhNi8o~|8!{xCn8g=`f@H7<t4
zKg;w|6%M8+*XK>M0Q=LUHZZ-gC6FX*yQj9Z%~UPtb_KdQ9ipg?#44rRT3wHBSMwWm
zaq;-K)qUr_X8<}4j8P!x|1FRn=dq&m`1+IlB$ZKt1A%0X3f)Di+6qo2omfdH{q6$k
zEorNUlpl3#96C0yy($#GwtjUA-J_R<XTaXM(GJ?)g3h@F=$3nGP{rI8DVmI{1d5MN
z{&eqt9!)|~Fg`Fvg8gvx#mbvYH|}+@r+2|l*EP-SnX56zv|NPsl@!UQ()8tcs`i7X
zr=8NtJtrT<0l4OF3sw)XpKN0N{l|BqeBEmprY_K)-GuveO_S$XvUmUA{XK&yYX1RE
CowjrU

literal 0
HcmV?d00001

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data
new file mode 100644
index 0000000000000000000000000000000000000000..294f4016d05bdd696670c4840f1f36a71f9239de
GIT binary patch
literal 16384
zcmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM
q7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjMyaxtA00031

literal 0
HcmV?d00001

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.index b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.index
new file mode 100644
index 0000000000000000000000000000000000000000..f7bb9ed1e3f24bf13bc8596347fb76e10b61d7a7
GIT binary patch
literal 473
zcmZ4UmVvd3fr0S?5O*s|N=yoO4EAJTU=Z;1@d43LzyhKL9DQ9}K@5Z(NF0|O2Oc>d
sJaPhf<V5huN#K!_!6T=DM@|KgoCY2_9XxUdc;rm*$XVc#v%!!9088&I&Hw-a

literal 0
HcmV?d00001

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
new file mode 100644
index 0000000000000..bc22293f99d4d
--- /dev/null
+++ b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
@@ -0,0 +1,4 @@
+#Key and value types
+#Wed Oct 11 21:36:14 WGST 2023
+valueType=org.opensearch.core.common.bytes.BytesReference
+keyType=org.opensearch.indices.IndicesRequestCache$Key
diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin
new file mode 100644
index 0000000000000000000000000000000000000000..ef98475e283358e20439f5c5846b5c71b003ab05
GIT binary patch
literal 1367
zcmeHG&ubGw6n;&tS}RyQi(Ud6NO!?%5M!-15*pcB%vKP4nrvRWlkU!}Gqakic=4b|
z!CnNtc=1=os~$v;g8mKd!Gj0!fADRxX;WHD@N5oCX4&t@`@VUrzhT%wp)A9QdFC>^
z#avXJ1^f=G@ws{nJ>wdMEG!31>#AsATPr9@wP>O5G0#Vfhf!cfs5I9Gg@;y!2Y8Wb
zY`BIQtYIWIH&QL<=Hn>9kuzV0K8>uz8OT&&m?w-5?Db{PKxH8B5Vf6B+dATQAG6&d
zoaua8YDYjnGBdHwj7~>jhD%}1D<(8YL4%N;F8bx#gSXGlmew;c<UlrLk%64kqUr6}
z@WA$@@M5J1W0yDn&a#L^NwBz0kPCxKu)x{`TARQbrhNi8o~|8!{xCn8g=`f@H7<t4
zKg;w|6%M8+*XK>M0Q=LUHZZ-gC6FX*yQj9Z%~UPtb_KdQ9ipg?#44rRT3wHBSMwWm
zaq;-K)qUr_X8<}4j8P!x|1FRn=dq&m`1+IlB$ZKt1A%0X3f)Di+6qo2omfdH{q6$k
zEorNUlpl3#96C0yy($#GwtjUA-J_R<XTaXM(GJ?)g3h@F=$3nGP{rI8DVmI{1d5MN
z{&eqt9!)|~Fg`Fvg8gvx#mbvYH|}+@r+2|l*EP-SnX56zv|NPsl@!UQ()8tcs`i7X
zr=8NtJtrT<0l4OF3sw)XpKN0N{l|BqeBEmprY_K)-GuveO_S$XvUmUA{XK&yYX1RE
CowjrU

literal 0
HcmV?d00001

diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 2e69fd372c7c6..2e33eb3b05b64 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -198,6 +198,13 @@ public double getHitRatio() {
         return hits / (hits + ts.getMisses());
     }
 
+    public void close() {
+        // Call this method after each test, otherwise the directory will stay locked and you won't be able to
+        // initialize another IndicesRequestCache
+        cacheManager.close();
+    }
+
+
     // See https://stackoverflow.com/questions/45827753/listenerobject-not-found-in-imports-for-ehcache-3 for API reference
     // it's not actually documented by ehcache :(
     // This class is used to get the old value from mutating calls to the cache, and it uses those to create a RemovalNotification
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index d55c188b8a418..d92e139270dec 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -399,4 +399,8 @@ long count() {
     int numRegisteredCloseListeners() { // for testing
         return registeredClosedListeners.size();
     }
+
+    public void closeDiskTier() {
+        tieredCacheHandler.closeDiskTier();
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
index 5fe41f5adce94..0a70375790191 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
@@ -21,4 +21,6 @@ public interface TieredCacheHandler<K, V> {
     long count();
 
     CachingTier<K, V> getOnHeapCachingTier();
+
+    void closeDiskTier();
 }
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index b1b68bbf93f79..a2df5545eff6b 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -25,7 +25,7 @@
 public class TieredCacheSpilloverStrategyHandler<K, V> implements TieredCacheHandler<K, V>, RemovalListener<K, V> {
 
     private final OnHeapCachingTier<K, V> onHeapCachingTier;
-    private final CachingTier<K, V> diskCachingTier;
+    private final EhcacheDiskCachingTier<K, V> diskCachingTier; // changed for testing
     private final TieredCacheEventListener<K, V> tieredCacheEventListener;
 
     /**
@@ -35,7 +35,8 @@ public class TieredCacheSpilloverStrategyHandler<K, V> implements TieredCacheHan
 
     private TieredCacheSpilloverStrategyHandler(
         OnHeapCachingTier<K, V> onHeapCachingTier,
-        CachingTier<K, V> diskCachingTier,
+        EhcacheDiskCachingTier<K, V> diskCachingTier,
+        // changed to EhcacheDiskCachingTier from CachingTier, to enable close() method, which is needed for tests. Implement close() in CachingTier or DiskCachingTier or something?
         TieredCacheEventListener<K, V> tieredCacheEventListener
     ) {
         this.onHeapCachingTier = Objects.requireNonNull(onHeapCachingTier);
@@ -127,6 +128,10 @@ private Function<K, CacheValue<V>> getValueFromTierCache() {
             return null;
         };
     }
+    @Override
+    public void closeDiskTier() {
+        diskCachingTier.close();
+    }
 
     public static class CacheValue<V> {
         V value;
@@ -163,7 +168,7 @@ public Builder<K, V> setTieredCacheEventListener(TieredCacheEventListener<K, V>
         public TieredCacheSpilloverStrategyHandler<K, V> build() {
             return new TieredCacheSpilloverStrategyHandler<K, V>(
                 this.onHeapCachingTier,
-                this.diskCachingTier,
+                (EhcacheDiskCachingTier<K, V>) this.diskCachingTier, // not sure why it was yelling about this, it already is an EhcacheDiskCachingTier
                 this.tieredCacheEventListener
             );
         }
diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy
index 20e5671d582a6..1fbfbb323e3af 100644
--- a/server/src/main/resources/org/opensearch/bootstrap/security.policy
+++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy
@@ -194,6 +194,8 @@ grant {
   permission java.lang.RuntimePermission "getenv.*";
   permission java.io.FilePermission "disk_cache_tier", "read"; // change this to wherever we will put disk tier folder
   permission java.io.FilePermission "disk_cache_tier", "write";
+  permission java.io.FilePermission "disk_cache_tier", "delete";
   permission java.io.FilePermission "disk_cache_tier/-", "read";
   permission java.io.FilePermission "disk_cache_tier/-", "write";
+  permission java.io.FilePermission "disk_cache_tier/-", "delete";
 };
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 8494259c8fd8a..5318a4f16d8f3 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -119,6 +119,7 @@ public void testBasicOperationsCache() throws Exception {
 
         IOUtils.close(reader, writer, dir, cache);
         assertEquals(0, cache.numRegisteredCloseListeners());
+        cache.closeDiskTier();
     }
 
     public void testCacheDifferentReaders() throws Exception {
@@ -213,6 +214,7 @@ public void testCacheDifferentReaders() throws Exception {
 
         IOUtils.close(secondReader, writer, dir, cache);
         assertEquals(0, cache.numRegisteredCloseListeners());
+        cache.closeDiskTier();
     }
 
     public void testEviction() throws Exception {
@@ -242,6 +244,7 @@ public void testEviction() throws Exception {
             assertEquals("bar", value2.streamInput().readString());
             size = requestCacheStats.stats().getMemorySize();
             IOUtils.close(reader, secondReader, writer, dir, cache);
+            cache.closeDiskTier();
         }
         IndicesRequestCache cache = new IndicesRequestCache(
             Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), size.getBytes() + 1 + "b").build()
@@ -278,6 +281,7 @@ public void testEviction() throws Exception {
         assertEquals(2, cache.count());
         assertEquals(1, requestCacheStats.stats().getEvictions());
         IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
+        cache.closeDiskTier();
     }
 
     public void testClearAllEntityIdentity() throws Exception {
@@ -325,6 +329,7 @@ public void testClearAllEntityIdentity() throws Exception {
         assertEquals("baz", value3.streamInput().readString());
 
         IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
+        cache.closeDiskTier();
 
     }
 
@@ -430,6 +435,7 @@ public void testInvalidate() throws Exception {
 
         IOUtils.close(reader, writer, dir, cache);
         assertEquals(0, cache.numRegisteredCloseListeners());
+        cache.closeDiskTier();
     }
 
     public void testEqualsKey() throws IOException {

From 354c311dbde2fa96420da2c17920bc503dbcd815 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 12 Oct 2023 15:11:50 -0700
Subject: [PATCH 016/120] misc stuff for testing

---
 server/disk_cache_tier/.lock                  |   0
 ...tJavaSerializer-ObjectStreamClassIndex.bin | Bin 1367 -> 3081 bytes
 .../ehcache-disk-store.index                  | Bin 473 -> 473 bytes
 .../ehcache-disk-store.meta                   |   2 +-
 .../cache/request/ShardRequestCache.java      |  31 +++++-
 .../indices/EhcacheDiskCachingTier.java       |  93 ++++++++++++------
 .../indices/IndicesRequestCache.java          |   6 +-
 .../TieredCacheSpilloverStrategyHandler.java  |   4 +
 .../indices/IndicesRequestCacheTests.java     |  69 +++++++++++++
 9 files changed, 167 insertions(+), 38 deletions(-)
 delete mode 100644 server/disk_cache_tier/.lock

diff --git a/server/disk_cache_tier/.lock b/server/disk_cache_tier/.lock
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/key-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/key-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin
index ef98475e283358e20439f5c5846b5c71b003ab05..a8f701c5f90c8437b80ab6beaadff9b5cc26b90b 100644
GIT binary patch
literal 3081
zcmeHJU1%It6u#L_()?H(;~!|-h^)4ebOwzHiJP`I+k~!e65NeN%+t-x+1*KI=1%Xu
z)9h9eMQjBT1mlCEMMS9Dh%Y|G2T?E<p+5NHqp7G+L<HY_kb>vl-PxZeh9E|Ka36Me
z?ws?TbH4L;SAHdJk_@Dn7f?$+LQm5iaEH3i8Q_Oz<`=+{Imv-~DVGY7_H8H$PLhll
z3@BREDuUs7folZ7g(D>R78sL`3*%IPos(2T76K+5$#^+28wM_Pj(pzoY3FhnBe67T
za};8Rv=^Cc1DB*fgV1Ib+7y9fHXxfC@PrmkmI8twbY^gaGol&-l@9ZbSx$Z7fRBNo
zI~DB<>(5_&^P%LG7-`9nxJLs?5}5^z-VCMVnilh|kaN^AQ?<WSG(e+haCn13$Cto;
z>Kb)Lqpo;@iXtl8udnyMb*}$#oW#?lTR_1>{3$9H)8vNgQ?BUb<s9j#J_<>W%{39C
zxHcneo4Z-HmdI5MRCwBhRP7CU6|OmT5!{x}*Wkk81D98JjeQ*>L}#D}2|4oj2#GNJ
zhBL1`@oN8JOvWVXav8NJu@)IGadJ@Q#85eD^v-40qA;?6{9$fo1|Rc}zc(=O)xs}2
zp{=6qTauWYXH;vOP1=bZDpZpD;C3x0rkS6o4p*x1;saN6Pkq*{Na5trwjq@6>i&0@
zKREv6(-qyJEp(5~3yjy2kA)CobD?gLGhcNoMxP3gW`D%!v8RNpbM#|)8(1?-KAm@=
z5p3|vN1uG?vG=14lcNt~pPeGlV414#9@TgMpKZYbCpT&8UFsK%G|q+s@ZKNZe)sq1
z&W%)3beg)MN>NW;ZaVblfuCJ@ZD05D_b=7R=fqIm#RBjJP;M0s9GQ*<0)h2!7EXmA
zWJ=BA#G_E2T$4YqE%#oEkvPs(Y!OG6B^l9Uq>h3^L~EOyp=hiLt(nY_8*%*OY}SW}
zFNvYCC@zd5m~Lo+4+;)S3JgkFEuM(v5ur_oE6M%WJ7F;nIK@1V`R06Cf-uL`cNX%1
z!`A}UlB(nevEn*7>8lrXin%WCCY<jgG2sQWoWbpLX7%=u$1a>-*W}d{TR6E(D>}@W
zW-{7us_Z9976c09;K>rXdiJ)VN)zsj#2Xi2P1gS6r^S~?|G?d$n*HWTHa%05aGw=6
z-1FPTMhRUDHHmMo^&Pn*xp!@Y#4NV-l5*|aOG5`nmsT&wNh=a)(NKUSw^k;a!ZK@;
zNbqhOBz4>=5P}v^uD3xB*SF&rl5?aB=Yq9}+t5OZ{c}ws(TfQe)K)^sTSOQH909?u
zd5!z1r(vcl${RBj<K#ceoT{Pm>uFMlsU}*~$??AkUA>!VD98A}n))3Gpfw8MAE?Y7
zaIdY&y;`wv(iS^lyq!l4VrX{o=2-2Vs6k?r_l=ERyL(hMaD7JY;^a;}4>s4LonO9j
z@9=jo#7M_DY0oq6QJKaYkK;pS`e(=<OWm_Gcn1PLO+8S-rEmt@fG<=d>w9?hv?Gr<
h?FOkX4`h+qO()C_n`Cx-`sAd#uQOi*m4YBB{R#TT{U-nb

delta 11
ScmeB_xX!g9gq69VpaK9GQUm7z

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.index b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.index
index f7bb9ed1e3f24bf13bc8596347fb76e10b61d7a7..05ab5af78b038e9fc54d3d0ee751c5b0e700edd0 100644
GIT binary patch
delta 26
gcmcb~e3Myl&07Z6Dh39|3qahh`u?@-M!^$|0C#N&<^TWy

delta 26
gcmcb~e3Myl&07Z6Dh39|3qahhC@C>%qu>cf0B*_$t^fc4

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
index bc22293f99d4d..acd184629f922 100644
--- a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
+++ b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
@@ -1,4 +1,4 @@
 #Key and value types
-#Wed Oct 11 21:36:14 WGST 2023
+#Thu Oct 12 16:18:26 EDT 2023
 valueType=org.opensearch.core.common.bytes.BytesReference
 keyType=org.opensearch.indices.IndicesRequestCache$Key
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index 1beef5217355f..f09d2504680c7 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -56,11 +56,34 @@ public ShardRequestCache() {
 
     public RequestCacheStats stats() {
         // TODO: Change RequestCacheStats to support disk tier stats.
+        return stats(TierType.ON_HEAP);
+    }
+
+    public RequestCacheStats stats(TierType tierType) {
+        return new RequestCacheStats(
+            statsHolder.get(tierType).totalMetric.count(),
+            statsHolder.get(tierType).evictionsMetric.count(),
+            statsHolder.get(tierType).hitCount.count(),
+            statsHolder.get(tierType).missCount.count()
+        );
+    }
+
+    public RequestCacheStats overallStats() {
+        long totalSize = 0;
+        long totalEvictions = 0;
+        long totalHits = 0;
+        long totalMisses = 0;
+        for (TierType tierType : TierType.values()) {
+            totalSize += statsHolder.get(tierType).totalMetric.count();
+            totalEvictions += statsHolder.get(tierType).evictionsMetric.count();
+            totalHits += statsHolder.get(tierType).hitCount.count();
+            totalMisses += statsHolder.get(tierType).missCount.count();
+        }
         return new RequestCacheStats(
-            statsHolder.get(TierType.ON_HEAP).totalMetric.count(),
-            statsHolder.get(TierType.ON_HEAP).evictionsMetric.count(),
-            statsHolder.get(TierType.ON_HEAP).hitCount.count(),
-            statsHolder.get(TierType.ON_HEAP).missCount.count()
+            totalSize,
+            totalEvictions,
+            totalHits,
+            totalMisses
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 2e33eb3b05b64..8fc62fe02bd9a 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.indices;
 
+import org.ehcache.PersistentCacheManager;
 import org.ehcache.config.builders.CacheConfigurationBuilder;
 import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
 import org.ehcache.config.builders.CacheManagerBuilder;
@@ -16,7 +17,6 @@
 import org.ehcache.config.units.MemoryUnit;
 import org.ehcache.core.internal.statistics.DefaultStatisticsService;
 import org.ehcache.core.spi.service.StatisticsService;
-import org.ehcache.core.statistics.CacheStatistics;
 import org.ehcache.core.statistics.TierStatistics;
 import org.ehcache.event.CacheEvent;
 import org.ehcache.event.CacheEventListener;
@@ -25,7 +25,6 @@
 import org.opensearch.common.ExponentiallyWeightedMovingAverage;
 import org.opensearch.common.cache.RemovalListener;
 import org.ehcache.Cache;
-import org.ehcache.CacheManager;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
 
@@ -33,55 +32,57 @@
 
 public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalListener<K, V> {
 
-    private CacheManager cacheManager;
-    private final Cache<K, V> cache;
+    private final PersistentCacheManager cacheManager;
+    public final Cache<K, V> cache; // make private after debug
 
     private final Class<K> keyType; // I think these are needed to pass to newCacheConfigurationBuilder
     private final Class<V> valueType;
     private final String DISK_CACHE_FP = "disk_cache_tier"; // this should probably be defined somewhere else since we need to change security.policy based on its value
     private RemovalListener<K, V> removalListener;
-    private final CacheStatistics cacheStats;
+    private StatisticsService statsService; // non functional
     private ExponentiallyWeightedMovingAverage getTimeMillisEWMA;
     private static final double GET_TIME_EWMA_ALPHA  = 0.3; // This is the value used elsewhere in OpenSearch
     private static final int MIN_WRITE_THREADS = 0;
     private static final int MAX_WRITE_THREADS = 4; // Max number of threads for the PooledExecutionService which handles writes
+    private static final String cacheAlias = "diskTier";
+    private final boolean isPersistent;
+    private int count; // number of entries in cache
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
     // private IndicesRequestCacheDiskTierPolicy policy;
 
-    public EhcacheDiskCachingTier(long maxWeightInBytes, long maxKeystoreWeightInBytes, Class<K> keyType, Class<V> valueType) {
+    public EhcacheDiskCachingTier(boolean isPersistent, long maxWeightInBytes, long maxKeystoreWeightInBytes, Class<K> keyType, Class<V> valueType) {
         this.keyType = keyType;
         this.valueType = valueType;
-        String cacheAlias = "diskTier";
-        StatisticsService statsService = new DefaultStatisticsService();
+        this.isPersistent = isPersistent;
+        this.count = 0;
+        statsService = new DefaultStatisticsService();
 
-        // our EhcacheEventListener should receive events every time an entry is removed, but not when it's created
+        // our EhcacheEventListener should receive events every time an entry is changed
         CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
-            .newEventListenerConfiguration(new EhcacheEventListener(this),
+            .newEventListenerConfiguration(new EhcacheEventListener(this, this),
                 EventType.EVICTED,
                 EventType.EXPIRED,
                 EventType.REMOVED,
-                EventType.UPDATED)
-            .ordered().asynchronous(); // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
+                EventType.UPDATED,
+                EventType.CREATED);
+            //.ordered().asynchronous(); // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
 
         PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
             .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
             .build();
 
         this.cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
-            .using(statsService)
+            .using(statsService) // https://stackoverflow.com/questions/40453859/how-to-get-ehcache-3-1-statistics
             .using(threadConfig)
             .with(CacheManagerBuilder.persistence(DISK_CACHE_FP))
             .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                keyType, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, true))
+                keyType, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, isPersistent))
                 .withService(listenerConfig) // stackoverflow shows .add(), but IDE says this is deprecated. idk
             ).build(true);
         this.cache = cacheManager.getCache(cacheAlias, keyType, valueType);
-        this.cacheStats = statsService.getCacheStatistics(cacheAlias);
         this.getTimeMillisEWMA = new ExponentiallyWeightedMovingAverage(GET_TIME_EWMA_ALPHA, 10);
 
-        // try and feed it an OpenSearch threadpool rather than its default ExecutionService?
-
         // this.keystore = new RBMIntKeyLookupStore((int) Math.pow(2, 28), maxKeystoreWeightInBytes);
         // this.policies = new CacheTierPolicy[]{ new IndicesRequestCacheTookTimePolicy(settings, clusterSettings) };
         // this.policy = new IndicesRequestCacheDiskTierPolicy(this.policies, true);
@@ -109,6 +110,7 @@ public void put(K key, V value) {
         // CheckDataResult policyResult = policy.checkData(value)
         // if (policyResult.isAccepted()) {
         cache.put(key, value);
+        //count++;
         // keystore.add(key.hashCode());
         // else { do something with policyResult.deniedReason()? }
         // }
@@ -126,6 +128,7 @@ public void invalidate(K key) {
 
         // if (keystore.contains(key.hashCode()) {
         cache.remove(key);
+        //count--;
         // keystore.remove(key.hashCode());
         // }
     }
@@ -154,8 +157,15 @@ public Iterable<K> keys() {
 
     @Override
     public int count() {
-        // this might be an expensive disk-seek call. Might be better to keep track ourselves?
-        return (int) getTierStats().getMappings();
+        return count;
+        //return (int) getTierStats().getMappings();
+    }
+
+    protected void countInc() {
+        count++;
+    }
+    protected void countDec() {
+        count--;
     }
 
     @Override
@@ -172,8 +182,11 @@ public double getTimeMillisEWMA() {
         return getTimeMillisEWMA.getAverage();
     }
 
-    private TierStatistics getTierStats() {
-        return cacheStats.getTierStatistics().get("Disk");
+    // these aren't really needed, ShardRequestCache handles it
+    // Also, it seems that ehcache doesn't have functioning statistics anyway!
+
+    /*public TierStatistics getTierStats() {
+        return statsService.getCacheStatistics(cacheAlias).getTierStatistics().get("Disk");
     }
 
     public long getHits() {
@@ -196,11 +209,16 @@ public double getHitRatio() {
         TierStatistics ts = getTierStats();
         long hits = ts.getHits();
         return hits / (hits + ts.getMisses());
+    }*/
+
+    public boolean isPersistent() {
+        return isPersistent;
     }
 
     public void close() {
         // Call this method after each test, otherwise the directory will stay locked and you won't be able to
-        // initialize another IndicesRequestCache
+        // initialize another IndicesRequestCache (for example in the next test that runs)
+        cacheManager.removeCache(cacheAlias);
         cacheManager.close();
     }
 
@@ -208,20 +226,36 @@ public void close() {
     // See https://stackoverflow.com/questions/45827753/listenerobject-not-found-in-imports-for-ehcache-3 for API reference
     // it's not actually documented by ehcache :(
     // This class is used to get the old value from mutating calls to the cache, and it uses those to create a RemovalNotification
-    private class EhcacheEventListener implements CacheEventListener<Object, Object> { // try making these specific, but i dont think itll work
+    // It also handles incrementing and decrementing the count for the disk tier, since ehcache's statistics functionality
+    // does not seem to work
+    private class EhcacheEventListener implements CacheEventListener<K, V> { // try making these specific, but i dont think itll work
         private RemovalListener<K, V> removalListener;
-        EhcacheEventListener(RemovalListener<K, V> removalListener) {
+        private EhcacheDiskCachingTier<K, V> tier;
+        EhcacheEventListener(RemovalListener<K, V> removalListener, EhcacheDiskCachingTier<K, V> tier) {
             this.removalListener = removalListener;
+            this.tier = tier; // needed to handle count changes
         }
         @Override
-        public void onEvent(CacheEvent<?, ?> event) {
-            // send a RemovalNotification
-            K key = (K) event.getKey(); // I think these casts should be ok?
-            V oldValue = (V) event.getOldValue();
-            V newValue = (V) event.getNewValue();
+        public void onEvent(CacheEvent<? extends K, ? extends V> event) {
+            K key = event.getKey();
+            V oldValue = event.getOldValue();
+            V newValue = event.getNewValue();
             EventType eventType = event.getType();
+
+            System.out.println("I am eventing!!");
+
+            // handle changing count for the disk tier
+            if (oldValue == null && newValue != null) {
+                tier.countInc();
+            } else if (oldValue != null && newValue == null) {
+                tier.countDec();
+            }
+
+            // handle creating a RemovalReason, unless eventType is CREATED
             RemovalReason reason;
             switch (eventType) {
+                case CREATED:
+                    return;
                 case EVICTED:
                     reason = RemovalReason.EVICTED; // why is there both RemovalReason.EVICTED and RemovalReason.CAPACITY?
                     break;
@@ -236,7 +270,6 @@ public void onEvent(CacheEvent<?, ?> event) {
                 default:
                     reason = null;
             }
-            // we don't subscribe to CREATED type, which is the only other option
             removalListener.onRemoval(new RemovalNotification<K, V>(key, oldValue, reason));
         }
     }
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index d92e139270dec..9a475be609a1a 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -107,8 +107,8 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
     private final TimeValue expire;
     // private final Cache<Key, BytesReference> cache;
 
-    private final TieredCacheHandler<Key, BytesReference> tieredCacheHandler;
-
+    //private final TieredCacheHandler<Key, BytesReference> tieredCacheHandler; // made public TieredCacheSpilloverStrategyHandler for testing
+    public final TieredCacheSpilloverStrategyHandler<Key, BytesReference> tieredCacheHandler;
     IndicesRequestCache(Settings settings) {
         this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
         this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
@@ -130,7 +130,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
 
         // changed to Integer for testing of bulk writes
         EhcacheDiskCachingTier<Key, BytesReference> diskCachingTier;
-        diskCachingTier = new EhcacheDiskCachingTier<>(diskTierWeight, 0, Key.class, BytesReference.class);
+        diskCachingTier = new EhcacheDiskCachingTier<>(true, diskTierWeight, 0, Key.class, BytesReference.class);
         tieredCacheHandler = new TieredCacheSpilloverStrategyHandler.Builder<Key, BytesReference>().setOnHeapCachingTier(
             openSearchOnHeapCache
         ).setOnDiskCachingTier(diskCachingTier).setTieredCacheEventListener(this).build();
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index a2df5545eff6b..5657e3519f2ca 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -109,6 +109,10 @@ public CachingTier<K, V> getOnHeapCachingTier() {
         return this.onHeapCachingTier;
     }
 
+    public EhcacheDiskCachingTier<K, V> getDiskCachingTier() { // change to CachingTier after debug
+        return this.diskCachingTier;
+    }
+
     private void setRemovalListeners() {
         for (CachingTier<K, V> cachingTier : cachingTierList) {
             cachingTier.setRemovalListener(this);
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 5318a4f16d8f3..e8ad68c67d9c8 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -45,6 +45,7 @@
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.ehcache.core.statistics.TierStatistics;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
@@ -122,6 +123,74 @@ public void testBasicOperationsCache() throws Exception {
         cache.closeDiskTier();
     }
 
+    public void testAddDirectToEhcache() throws Exception {
+        ShardRequestCache requestCacheStats = new ShardRequestCache();
+        Settings.Builder settingsBuilder = Settings.builder();
+        long heapSizeBytes = 1000;
+        settingsBuilder.put("indices.requests.cache.size", new ByteSizeValue(heapSizeBytes));
+        IndicesRequestCache cache = new IndicesRequestCache(settingsBuilder.build());
+
+        // set up a key
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+        writer.addDocument(newDoc(0, "foo"));
+        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        AtomicBoolean indexShard = new AtomicBoolean(true);
+        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+        Loader loader = new Loader(reader, 0);
+        IndicesRequestCache.Key[] keys = new IndicesRequestCache.Key[9];
+        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
+        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(entity, reader.getReaderCacheHelper().getKey(), termBytes);
+
+        TestBytesReference value = new TestBytesReference(124);
+        cache.tieredCacheHandler.getDiskCachingTier().cache.put(key, value);
+
+        IOUtils.close(reader, writer, dir, cache);
+        cache.closeDiskTier();
+    }
+
+    public void testSpillover() throws Exception {
+        // fill the on-heap cache until we spill over
+        ShardRequestCache requestCacheStats = new ShardRequestCache();
+        Settings.Builder settingsBuilder = Settings.builder();
+        long heapSizeBytes = 1000; // each of these queries is 115 bytes, so we can fit 8 in the heap cache
+        settingsBuilder.put("indices.requests.cache.size", new ByteSizeValue(heapSizeBytes));
+        IndicesRequestCache cache = new IndicesRequestCache(settingsBuilder.build());
+
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+        writer.addDocument(newDoc(0, "foo"));
+        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        AtomicBoolean indexShard = new AtomicBoolean(true);
+
+        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+        Loader loader = new Loader(reader, 0);
+        System.out.println("On-heap cache size at start = " + requestCacheStats.stats().getMemorySizeInBytes());
+        IndicesRequestCache.Key[] keys = new IndicesRequestCache.Key[9];
+        for (int i = 0; i < 9; i++) {
+            TermQueryBuilder termQuery = new TermQueryBuilder("id", String.valueOf(i));
+            BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+            keys[i] = new IndicesRequestCache.Key(entity, reader.getReaderCacheHelper().getKey(), termBytes);
+            BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
+            System.out.println("On-heap cache size after " + (i+1) + " queries = " + requestCacheStats.stats().getMemorySizeInBytes());
+            System.out.println("Disk cache size after " + (i+1) + " queries = " + requestCacheStats.stats(TierType.DISK).getMemorySizeInBytes());
+        }
+        // attempt to get value from disk cache, the first key should have been evicted
+        BytesReference firstValue = cache.tieredCacheHandler.get(keys[0]);
+        System.out.println("Final on-heap cache size = " + requestCacheStats.stats().getMemorySizeInBytes()); // is correctly 920
+        //System.out.println("Final self-reported disk size = " + cache.tieredCacheHandler.getDiskWeightBytes()); // is 0, should be 115
+        System.out.println("On-heap tier evictions = " + requestCacheStats.stats().getEvictions()); // is correctly 1
+        System.out.println("Disk tier hits = " + requestCacheStats.stats(TierType.DISK).getHitCount()); // should be 1, is 0 bc keys not serializable
+        System.out.println("Disk tier misses = " + requestCacheStats.stats(TierType.DISK).getMissCount()); // should be 9, is 10 bc keys not serializable
+        //System.out.println("Disk tier self-reported misses = " + cache.tieredCacheHandler.getDiskCachingTier().getMisses()); // should be same as other one
+        System.out.println("On-heap tier hits = " + requestCacheStats.stats().getHitCount()); // is correctly 0
+        System.out.println("On-heap tier misses = " + requestCacheStats.stats().getMissCount()); // is correctly 10
+        System.out.println("Disk count = " + cache.tieredCacheHandler.getDiskCachingTier().count()); // should be 1, is 0
+        IOUtils.close(reader, writer, dir, cache);
+        cache.closeDiskTier();
+    }
+
     public void testCacheDifferentReaders() throws Exception {
         IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
         AtomicBoolean indexShard = new AtomicBoolean(true);

From 66915b6b1a7f79bc8724fc5f3df5ab7ab9bf4e31 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Fri, 13 Oct 2023 10:07:01 -0700
Subject: [PATCH 017/120] Moved more stuff around for debugging issues

---
 .../indices/EhcacheDiskCachingTier.java       | 61 ++------------
 .../indices/EhcacheEventListener.java         | 68 +++++++++++++++
 .../indices/IndicesRequestCacheTests.java     | 83 ++++++++++++++++++-
 3 files changed, 159 insertions(+), 53 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/indices/EhcacheEventListener.java

diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 8fc62fe02bd9a..0ebacae9562c3 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -27,6 +27,7 @@
 import org.ehcache.Cache;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.metrics.CounterMetric;
 
 import java.util.Collections;
 
@@ -37,7 +38,7 @@ public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalL
 
     private final Class<K> keyType; // I think these are needed to pass to newCacheConfigurationBuilder
     private final Class<V> valueType;
-    private final String DISK_CACHE_FP = "disk_cache_tier"; // this should probably be defined somewhere else since we need to change security.policy based on its value
+    public final static String DISK_CACHE_FP = "disk_cache_tier"; // this should probably be defined somewhere else since we need to change security.policy based on its value
     private RemovalListener<K, V> removalListener;
     private StatisticsService statsService; // non functional
     private ExponentiallyWeightedMovingAverage getTimeMillisEWMA;
@@ -46,7 +47,7 @@ public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalL
     private static final int MAX_WRITE_THREADS = 4; // Max number of threads for the PooledExecutionService which handles writes
     private static final String cacheAlias = "diskTier";
     private final boolean isPersistent;
-    private int count; // number of entries in cache
+    private CounterMetric count; // number of entries in cache
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
     // private IndicesRequestCacheDiskTierPolicy policy;
@@ -55,12 +56,12 @@ public EhcacheDiskCachingTier(boolean isPersistent, long maxWeightInBytes, long
         this.keyType = keyType;
         this.valueType = valueType;
         this.isPersistent = isPersistent;
-        this.count = 0;
+        this.count = new CounterMetric();
         statsService = new DefaultStatisticsService();
 
         // our EhcacheEventListener should receive events every time an entry is changed
         CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
-            .newEventListenerConfiguration(new EhcacheEventListener(this, this),
+            .newEventListenerConfiguration(new EhcacheEventListener<K, V>(this, this.count),
                 EventType.EVICTED,
                 EventType.EXPIRED,
                 EventType.REMOVED,
@@ -157,15 +158,15 @@ public Iterable<K> keys() {
 
     @Override
     public int count() {
-        return count;
+        return (int) count.count();
         //return (int) getTierStats().getMappings();
     }
 
     protected void countInc() {
-        count++;
+        count.inc();
     }
     protected void countDec() {
-        count--;
+        count.dec();
     }
 
     @Override
@@ -228,49 +229,5 @@ public void close() {
     // This class is used to get the old value from mutating calls to the cache, and it uses those to create a RemovalNotification
     // It also handles incrementing and decrementing the count for the disk tier, since ehcache's statistics functionality
     // does not seem to work
-    private class EhcacheEventListener implements CacheEventListener<K, V> { // try making these specific, but i dont think itll work
-        private RemovalListener<K, V> removalListener;
-        private EhcacheDiskCachingTier<K, V> tier;
-        EhcacheEventListener(RemovalListener<K, V> removalListener, EhcacheDiskCachingTier<K, V> tier) {
-            this.removalListener = removalListener;
-            this.tier = tier; // needed to handle count changes
-        }
-        @Override
-        public void onEvent(CacheEvent<? extends K, ? extends V> event) {
-            K key = event.getKey();
-            V oldValue = event.getOldValue();
-            V newValue = event.getNewValue();
-            EventType eventType = event.getType();
-
-            System.out.println("I am eventing!!");
-
-            // handle changing count for the disk tier
-            if (oldValue == null && newValue != null) {
-                tier.countInc();
-            } else if (oldValue != null && newValue == null) {
-                tier.countDec();
-            }
-
-            // handle creating a RemovalReason, unless eventType is CREATED
-            RemovalReason reason;
-            switch (eventType) {
-                case CREATED:
-                    return;
-                case EVICTED:
-                    reason = RemovalReason.EVICTED; // why is there both RemovalReason.EVICTED and RemovalReason.CAPACITY?
-                    break;
-                case EXPIRED:
-                case REMOVED:
-                    reason = RemovalReason.INVALIDATED;
-                    // this is probably fine for EXPIRED. We use cache.remove() to invalidate keys, but this might overlap with RemovalReason.EXPLICIT?
-                    break;
-                case UPDATED:
-                    reason = RemovalReason.REPLACED;
-                    break;
-                default:
-                    reason = null;
-            }
-            removalListener.onRemoval(new RemovalNotification<K, V>(key, oldValue, reason));
-        }
-    }
+
 }
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
new file mode 100644
index 0000000000000..ee240467d0af9
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.ehcache.event.CacheEvent;
+import org.ehcache.event.CacheEventListener;
+import org.ehcache.event.EventType;
+import org.opensearch.common.cache.RemovalListener;
+import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.indices.EhcacheDiskCachingTier;
+
+// moved to another file for testing flexibility purposes
+
+public class EhcacheEventListener<K, V> implements CacheEventListener<K, V> { // make it private after debugging
+    private RemovalListener<K, V> removalListener;
+    private CounterMetric counter;
+    EhcacheEventListener(RemovalListener<K, V> removalListener, CounterMetric counter) {
+        this.removalListener = removalListener;
+        this.counter = counter; // needed to handle count changes
+    }
+    @Override
+    public void onEvent(CacheEvent<? extends K, ? extends V> event) {
+        K key = event.getKey();
+        V oldValue = event.getOldValue();
+        V newValue = event.getNewValue();
+        EventType eventType = event.getType();
+
+        System.out.println("I am eventing!!");
+
+        // handle changing count for the disk tier
+        if (oldValue == null && newValue != null) {
+            counter.inc();
+        } else if (oldValue != null && newValue == null) {
+            counter.dec();
+        } else {
+            int j;
+        }
+
+        // handle creating a RemovalReason, unless eventType is CREATED
+        RemovalReason reason;
+        switch (eventType) {
+            case CREATED:
+                return;
+            case EVICTED:
+                reason = RemovalReason.EVICTED; // why is there both RemovalReason.EVICTED and RemovalReason.CAPACITY?
+                break;
+            case EXPIRED:
+            case REMOVED:
+                reason = RemovalReason.INVALIDATED;
+                // this is probably fine for EXPIRED. We use cache.remove() to invalidate keys, but this might overlap with RemovalReason.EXPLICIT?
+                break;
+            case UPDATED:
+                reason = RemovalReason.REPLACED;
+                break;
+            default:
+                reason = null;
+        }
+        removalListener.onRemoval(new RemovalNotification<K, V>(key, oldValue, reason));
+    }
+}
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index e8ad68c67d9c8..375a77815b825 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -45,10 +45,25 @@
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.ehcache.Cache;
+import org.ehcache.PersistentCacheManager;
+import org.ehcache.config.builders.CacheConfigurationBuilder;
+import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
+import org.ehcache.config.builders.CacheManagerBuilder;
+import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
+import org.ehcache.config.builders.ResourcePoolsBuilder;
+import org.ehcache.config.units.MemoryUnit;
+import org.ehcache.core.internal.statistics.DefaultStatisticsService;
+import org.ehcache.core.spi.service.StatisticsService;
 import org.ehcache.core.statistics.TierStatistics;
+import org.ehcache.event.EventType;
+import org.ehcache.impl.config.executor.PooledExecutionServiceConfiguration;
 import org.opensearch.common.CheckedSupplier;
+import org.opensearch.common.cache.RemovalListener;
+import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
+import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.bytes.AbstractBytesReference;
@@ -62,6 +77,7 @@
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -150,6 +166,71 @@ public void testAddDirectToEhcache() throws Exception {
         cache.closeDiskTier();
     }
 
+    public void testSimpleEhcache() throws Exception {
+        // for debug only, delete
+         CounterMetric count = new CounterMetric();
+         String cacheAlias = "dummy";
+
+        class DummyRemovalListener implements RemovalListener<Integer, String> {
+            public DummyRemovalListener() { }
+            @Override
+            public void onRemoval(RemovalNotification<Integer, String> notification) {
+                System.out.println(":)");
+            }
+        }
+
+        CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
+            .newEventListenerConfiguration(new EhcacheEventListener<Integer, String>(new DummyRemovalListener(), count),
+                EventType.EVICTED,
+                EventType.EXPIRED,
+                EventType.REMOVED,
+                EventType.UPDATED,
+                EventType.CREATED)
+        .ordered().asynchronous(); // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
+
+        StatisticsService statsService = new DefaultStatisticsService();
+
+        PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
+            .defaultPool("default", 0, 4)
+            .build();
+
+        PersistentCacheManager cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
+            .using(statsService) // https://stackoverflow.com/questions/40453859/how-to-get-ehcache-3-1-statistics
+            .using(threadConfig)
+            .with(CacheManagerBuilder.persistence(EhcacheDiskCachingTier.DISK_CACHE_FP))
+            .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
+                    IndicesRequestCache.Key.class, String.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(10, MemoryUnit.MB, false))
+                .withService(listenerConfig) // stackoverflow shows .add(), but IDE says this is deprecated. idk
+            ).build(true);
+        Cache<IndicesRequestCache.Key, String> cache = cacheManager.getCache(cacheAlias, IndicesRequestCache.Key.class, String.class);
+
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+        writer.addDocument(newDoc(0, "foo"));
+        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        AtomicBoolean indexShard = new AtomicBoolean(true);
+        ShardRequestCache requestCacheStats = new ShardRequestCache();
+        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+        Loader loader = new Loader(reader, 0);
+        System.out.println("On-heap cache size at start = " + requestCacheStats.stats().getMemorySizeInBytes());
+        IndicesRequestCache.Key[] keys = new IndicesRequestCache.Key[9];
+        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
+        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(entity, reader.getReaderCacheHelper().getKey(), termBytes);
+
+        cache.put(key, "blorp");
+        System.out.println("Counter value = " + count.count());
+        String res = cache.get(key);
+        System.out.println("Got result " + res);
+
+        System.out.println("Counter value = " + count.count());
+        //System.out.println("Hits = " + statsService.getCacheStatistics(cacheAlias).getTierStatistics().get("Disk").getHits());
+
+        cacheManager.removeCache(cacheAlias);
+        cacheManager.close();
+        IOUtils.close(reader, writer, dir);
+    }
+
     public void testSpillover() throws Exception {
         // fill the on-heap cache until we spill over
         ShardRequestCache requestCacheStats = new ShardRequestCache();
@@ -534,7 +615,7 @@ public void testEqualsKey() throws IOException {
         assertNotEquals(key1, key5);
     }
 
-    private class TestBytesReference extends AbstractBytesReference {
+    private class TestBytesReference extends AbstractBytesReference implements Serializable {
 
         int dummyValue;
 

From f985e6371bccd08b2668855e15033dd5188764a3 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Fri, 13 Oct 2023 16:39:04 -0700
Subject: [PATCH 018/120] fixed various serialization issues relating to silent
 ehcache put() and get() failures

---
 server/build.gradle                           |  2 +-
 .../indices/DummySerializableKey.java         | 50 +++++++++++++
 .../indices/EhcacheEventListener.java         |  2 +-
 .../indices/IndicesRequestCacheTests.java     | 74 ++++++++++++++-----
 4 files changed, 107 insertions(+), 21 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/indices/DummySerializableKey.java

diff --git a/server/build.gradle b/server/build.gradle
index 77cc411677ac5..dc3fa97ea4ca6 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -159,7 +159,7 @@ dependencies {
   api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}"
 
   // ehcache
-  api "org.ehcache:ehcache:3.10.8"
+  api "org.ehcache:ehcache:3.10.8" // using 3.3.1 rather than 3.10.8 to test if dependency conflicts are causing my issues
   api "org.slf4j:slf4j-api:1.7.36"
 
   testImplementation(project(":test:framework")) {
diff --git a/server/src/main/java/org/opensearch/indices/DummySerializableKey.java b/server/src/main/java/org/opensearch/indices/DummySerializableKey.java
new file mode 100644
index 0000000000000..7f2888f6e65f7
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/DummySerializableKey.java
@@ -0,0 +1,50 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DummySerializableKey implements Serializable {
+    private Integer i;
+    private String s;
+    public DummySerializableKey(Integer i, String s) {
+        this.i = i;
+        this.s = s;
+    }
+
+    public int getI() {
+        return i;
+    }
+    public String getS() {
+        return s;
+    }
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) {
+            return true;
+        }
+        if (!(o instanceof DummySerializableKey)) {
+            return false;
+        }
+        DummySerializableKey other = (DummySerializableKey) o;
+        return Objects.equals(this.i, other.i) && this.s.equals(other.s);
+    }
+    @Override
+    public final int hashCode() {
+        int result = 11;
+        if (i != null) {
+            result = 31 * result + i.hashCode();
+        }
+        if (s != null) {
+            result = 31 * result + s.hashCode();
+        }
+        return result;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
index ee240467d0af9..c6d2790dda95f 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
@@ -41,7 +41,7 @@ public void onEvent(CacheEvent<? extends K, ? extends V> event) {
         } else if (oldValue != null && newValue == null) {
             counter.dec();
         } else {
-            int j;
+            int j; // breakpoint
         }
 
         // handle creating a RemovalReason, unless eventType is CREATED
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 375a77815b825..8cdf5635439af 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -78,7 +78,9 @@
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 public class IndicesRequestCacheTests extends OpenSearchTestCase {
@@ -194,17 +196,58 @@ public void onRemoval(RemovalNotification<Integer, String> notification) {
             .defaultPool("default", 0, 4)
             .build();
 
-        PersistentCacheManager cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
-            .using(statsService) // https://stackoverflow.com/questions/40453859/how-to-get-ehcache-3-1-statistics
-            .using(threadConfig)
-            .with(CacheManagerBuilder.persistence(EhcacheDiskCachingTier.DISK_CACHE_FP))
-            .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                    IndicesRequestCache.Key.class, String.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(10, MemoryUnit.MB, false))
-                .withService(listenerConfig) // stackoverflow shows .add(), but IDE says this is deprecated. idk
-            ).build(true);
-        Cache<IndicesRequestCache.Key, String> cache = cacheManager.getCache(cacheAlias, IndicesRequestCache.Key.class, String.class);
+        PersistentCacheManager cacheManager;
+
+        boolean doIntCache = false;
+
+        if (doIntCache) {
+            cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
+                .using(statsService) // https://stackoverflow.com/questions/40453859/how-to-get-ehcache-3-1-statistics
+                .using(threadConfig)
+                .with(CacheManagerBuilder.persistence(EhcacheDiskCachingTier.DISK_CACHE_FP))
+                .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
+                        Integer.class, String.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(10, MemoryUnit.MB, false))
+                    .withService(listenerConfig)
+                ).build(true);
+            Cache<Integer, String> integerCache = cacheManager.getCache(cacheAlias, Integer.class, String.class);
+
+            integerCache.put(0, "blorp");
+            System.out.println("Counter value = " + count.count());
+            String res = integerCache.get(0);
+            System.out.println("Got result " + res);
+            System.out.println("Counter value = " + count.count());
+        } else {
+            cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
+                .using(statsService) // https://stackoverflow.com/questions/40453859/how-to-get-ehcache-3-1-statistics
+                .using(threadConfig)
+                .with(CacheManagerBuilder.persistence(EhcacheDiskCachingTier.DISK_CACHE_FP))
+                .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
+                        DummySerializableKey.class, String.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(10, MemoryUnit.MB, false))
+                    .withService(listenerConfig)
+                ).build(true);
+            Cache<DummySerializableKey, String> cache = cacheManager.getCache(cacheAlias, DummySerializableKey.class, String.class);
+
+            DummySerializableKey key = new DummySerializableKey(Integer.valueOf(0), "blah");
+            cache.put(key, "blorp");
+            System.out.println("Counter value = " + count.count());
+            String res = cache.get(key);
+            System.out.println("Got result " + res);
+            System.out.println("Counter value = " + count.count());
+            TierStatistics ts = statsService.getCacheStatistics(cacheAlias).getTierStatistics().get("Disk");
+            System.out.println("self-reported count = " + ts.getMappings());
+            System.out.println("self-reported misses = " + ts.getMisses());
+            System.out.println("self-reported hits = " + ts.getHits());
+
+            List<Cache.Entry<DummySerializableKey, String>> foos = new ArrayList<>();
+            for(Cache.Entry<DummySerializableKey, String> entry : cache) {
+                foos.add(entry);
+            }
+            int j = 0;
+            j++;
+            System.out.println(j);
+        }
 
-        Directory dir = newDirectory();
+        /*Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
         writer.addDocument(newDoc(0, "foo"));
         DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
@@ -216,19 +259,12 @@ public void onRemoval(RemovalNotification<Integer, String> notification) {
         IndicesRequestCache.Key[] keys = new IndicesRequestCache.Key[9];
         TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
         BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        IndicesRequestCache.Key key = new IndicesRequestCache.Key(entity, reader.getReaderCacheHelper().getKey(), termBytes);
-
-        cache.put(key, "blorp");
-        System.out.println("Counter value = " + count.count());
-        String res = cache.get(key);
-        System.out.println("Got result " + res);
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(entity, reader.getReaderCacheHelper().getKey(), termBytes);*/
 
-        System.out.println("Counter value = " + count.count());
-        //System.out.println("Hits = " + statsService.getCacheStatistics(cacheAlias).getTierStatistics().get("Disk").getHits());
 
         cacheManager.removeCache(cacheAlias);
         cacheManager.close();
-        IOUtils.close(reader, writer, dir);
+        //IOUtils.close(reader, writer, dir);
     }
 
     public void testSpillover() throws Exception {

From ae300e9c3c789d8adf7605a2a18c0a9528352cf9 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 16 Oct 2023 10:03:50 -0700
Subject: [PATCH 019/120] Cleanup

---
 ...tJavaSerializer-ObjectStreamClassIndex.bin | Bin 3081 -> 0 bytes
 .../ehcache-disk-store.data                   | Bin 16384 -> 0 bytes
 .../ehcache-disk-store.index                  | Bin 473 -> 0 bytes
 .../ehcache-disk-store.meta                   |   4 --
 ...tJavaSerializer-ObjectStreamClassIndex.bin | Bin 1367 -> 0 bytes
 .../indices/EhcacheDiskCachingTier.java       |  55 +++---------------
 .../indices/IndicesRequestCache.java          |   5 +-
 .../TieredCacheSpilloverStrategyHandler.java  |   2 +-
 .../indices/IndicesRequestCacheTests.java     |   7 ++-
 9 files changed, 17 insertions(+), 56 deletions(-)
 delete mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/key-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin
 delete mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data
 delete mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.index
 delete mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
 delete mode 100644 server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/key-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/key-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin
deleted file mode 100644
index a8f701c5f90c8437b80ab6beaadff9b5cc26b90b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3081
zcmeHJU1%It6u#L_()?H(;~!|-h^)4ebOwzHiJP`I+k~!e65NeN%+t-x+1*KI=1%Xu
z)9h9eMQjBT1mlCEMMS9Dh%Y|G2T?E<p+5NHqp7G+L<HY_kb>vl-PxZeh9E|Ka36Me
z?ws?TbH4L;SAHdJk_@Dn7f?$+LQm5iaEH3i8Q_Oz<`=+{Imv-~DVGY7_H8H$PLhll
z3@BREDuUs7folZ7g(D>R78sL`3*%IPos(2T76K+5$#^+28wM_Pj(pzoY3FhnBe67T
za};8Rv=^Cc1DB*fgV1Ib+7y9fHXxfC@PrmkmI8twbY^gaGol&-l@9ZbSx$Z7fRBNo
zI~DB<>(5_&^P%LG7-`9nxJLs?5}5^z-VCMVnilh|kaN^AQ?<WSG(e+haCn13$Cto;
z>Kb)Lqpo;@iXtl8udnyMb*}$#oW#?lTR_1>{3$9H)8vNgQ?BUb<s9j#J_<>W%{39C
zxHcneo4Z-HmdI5MRCwBhRP7CU6|OmT5!{x}*Wkk81D98JjeQ*>L}#D}2|4oj2#GNJ
zhBL1`@oN8JOvWVXav8NJu@)IGadJ@Q#85eD^v-40qA;?6{9$fo1|Rc}zc(=O)xs}2
zp{=6qTauWYXH;vOP1=bZDpZpD;C3x0rkS6o4p*x1;saN6Pkq*{Na5trwjq@6>i&0@
zKREv6(-qyJEp(5~3yjy2kA)CobD?gLGhcNoMxP3gW`D%!v8RNpbM#|)8(1?-KAm@=
z5p3|vN1uG?vG=14lcNt~pPeGlV414#9@TgMpKZYbCpT&8UFsK%G|q+s@ZKNZe)sq1
z&W%)3beg)MN>NW;ZaVblfuCJ@ZD05D_b=7R=fqIm#RBjJP;M0s9GQ*<0)h2!7EXmA
zWJ=BA#G_E2T$4YqE%#oEkvPs(Y!OG6B^l9Uq>h3^L~EOyp=hiLt(nY_8*%*OY}SW}
zFNvYCC@zd5m~Lo+4+;)S3JgkFEuM(v5ur_oE6M%WJ7F;nIK@1V`R06Cf-uL`cNX%1
z!`A}UlB(nevEn*7>8lrXin%WCCY<jgG2sQWoWbpLX7%=u$1a>-*W}d{TR6E(D>}@W
zW-{7us_Z9976c09;K>rXdiJ)VN)zsj#2Xi2P1gS6r^S~?|G?d$n*HWTHa%05aGw=6
z-1FPTMhRUDHHmMo^&Pn*xp!@Y#4NV-l5*|aOG5`nmsT&wNh=a)(NKUSw^k;a!ZK@;
zNbqhOBz4>=5P}v^uD3xB*SF&rl5?aB=Yq9}+t5OZ{c}ws(TfQe)K)^sTSOQH909?u
zd5!z1r(vcl${RBj<K#ceoT{Pm>uFMlsU}*~$??AkUA>!VD98A}n))3Gpfw8MAE?Y7
zaIdY&y;`wv(iS^lyq!l4VrX{o=2-2Vs6k?r_l=ERyL(hMaD7JY;^a;}4>s4LonO9j
z@9=jo#7M_DY0oq6QJKaYkK;pS`e(=<OWm_Gcn1PLO+8S-rEmt@fG<=d>w9?hv?Gr<
h?FOkX4`h+qO()C_n`Cx-`sAd#uQOi*m4YBB{R#TT{U-nb

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.data
deleted file mode 100644
index 294f4016d05bdd696670c4840f1f36a71f9239de..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 16384
zcmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM
q7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjMyaxtA00031

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.index b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.index
deleted file mode 100644
index 05ab5af78b038e9fc54d3d0ee751c5b0e700edd0..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 473
zcmZ4UmVvd3fr0S?5O=G-e=QsC80^Wwz#!o1;{&3hfCWSgIQqJ{f*1%nkT@<m4m@%^
sc;p1|$cf;QlfWY<gGWvQkDLk~ISo8=I(Xy^@W`3qk+Z-fXM-UJ0DF)w1^@s6

diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
deleted file mode 100644
index acd184629f922..0000000000000
--- a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/offheap-disk-store/ehcache-disk-store.meta
+++ /dev/null
@@ -1,4 +0,0 @@
-#Key and value types
-#Thu Oct 12 16:18:26 EDT 2023
-valueType=org.opensearch.core.common.bytes.BytesReference
-keyType=org.opensearch.indices.IndicesRequestCache$Key
diff --git a/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin b/server/disk_cache_tier/file/diskTier_8a0816fa8975d40fe56fbdb0d44b485392d49431/value-serializer/holder-0-CompactJavaSerializer-ObjectStreamClassIndex.bin
deleted file mode 100644
index ef98475e283358e20439f5c5846b5c71b003ab05..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1367
zcmeHG&ubGw6n;&tS}RyQi(Ud6NO!?%5M!-15*pcB%vKP4nrvRWlkU!}Gqakic=4b|
z!CnNtc=1=os~$v;g8mKd!Gj0!fADRxX;WHD@N5oCX4&t@`@VUrzhT%wp)A9QdFC>^
z#avXJ1^f=G@ws{nJ>wdMEG!31>#AsATPr9@wP>O5G0#Vfhf!cfs5I9Gg@;y!2Y8Wb
zY`BIQtYIWIH&QL<=Hn>9kuzV0K8>uz8OT&&m?w-5?Db{PKxH8B5Vf6B+dATQAG6&d
zoaua8YDYjnGBdHwj7~>jhD%}1D<(8YL4%N;F8bx#gSXGlmew;c<UlrLk%64kqUr6}
z@WA$@@M5J1W0yDn&a#L^NwBz0kPCxKu)x{`TARQbrhNi8o~|8!{xCn8g=`f@H7<t4
zKg;w|6%M8+*XK>M0Q=LUHZZ-gC6FX*yQj9Z%~UPtb_KdQ9ipg?#44rRT3wHBSMwWm
zaq;-K)qUr_X8<}4j8P!x|1FRn=dq&m`1+IlB$ZKt1A%0X3f)Di+6qo2omfdH{q6$k
zEorNUlpl3#96C0yy($#GwtjUA-J_R<XTaXM(GJ?)g3h@F=$3nGP{rI8DVmI{1d5MN
z{&eqt9!)|~Fg`Fvg8gvx#mbvYH|}+@r+2|l*EP-SnX56zv|NPsl@!UQ()8tcs`i7X
zr=8NtJtrT<0l4OF3sw)XpKN0N{l|BqeBEmprY_K)-GuveO_S$XvUmUA{XK&yYX1RE
CowjrU

diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 0ebacae9562c3..2142b8f3b0930 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -34,7 +34,7 @@
 public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalListener<K, V> {
 
     private final PersistentCacheManager cacheManager;
-    public final Cache<K, V> cache; // make private after debug
+    private final Cache<K, V> cache; // make private after debug
 
     private final Class<K> keyType; // I think these are needed to pass to newCacheConfigurationBuilder
     private final Class<V> valueType;
@@ -48,6 +48,7 @@ public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalL
     private static final String cacheAlias = "diskTier";
     private final boolean isPersistent;
     private CounterMetric count; // number of entries in cache
+    private EhcacheEventListener listener;
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
     // private IndicesRequestCacheDiskTierPolicy policy;
@@ -57,17 +58,18 @@ public EhcacheDiskCachingTier(boolean isPersistent, long maxWeightInBytes, long
         this.valueType = valueType;
         this.isPersistent = isPersistent;
         this.count = new CounterMetric();
+        this.listener = new EhcacheEventListener<K, V>(this, this.count);
         statsService = new DefaultStatisticsService();
 
         // our EhcacheEventListener should receive events every time an entry is changed
         CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
-            .newEventListenerConfiguration(new EhcacheEventListener<K, V>(this, this.count),
+            .newEventListenerConfiguration(listener,
                 EventType.EVICTED,
                 EventType.EXPIRED,
                 EventType.REMOVED,
                 EventType.UPDATED,
-                EventType.CREATED);
-            //.ordered().asynchronous(); // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
+                EventType.CREATED)
+            .ordered().asynchronous(); // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
 
         PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
             .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
@@ -79,7 +81,7 @@ public EhcacheDiskCachingTier(boolean isPersistent, long maxWeightInBytes, long
             .with(CacheManagerBuilder.persistence(DISK_CACHE_FP))
             .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
                 keyType, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, isPersistent))
-                .withService(listenerConfig) // stackoverflow shows .add(), but IDE says this is deprecated. idk
+                .withService(listenerConfig)
             ).build(true);
         this.cache = cacheManager.getCache(cacheAlias, keyType, valueType);
         this.getTimeMillisEWMA = new ExponentiallyWeightedMovingAverage(GET_TIME_EWMA_ALPHA, 10);
@@ -91,7 +93,7 @@ public EhcacheDiskCachingTier(boolean isPersistent, long maxWeightInBytes, long
 
     @Override
     public V get(K key) {
-        // do we need to do the future stuff? I don't think so?
+        // I don't think we need to do the future stuff as the cache is threadsafe
 
         // if (keystore.contains(key.hashCode()) {
         long now = System.nanoTime();
@@ -101,7 +103,6 @@ public V get(K key) {
         return value;
         // }
         // return null;
-
     }
 
     @Override
@@ -111,7 +112,6 @@ public void put(K key, V value) {
         // CheckDataResult policyResult = policy.checkData(value)
         // if (policyResult.isAccepted()) {
         cache.put(key, value);
-        //count++;
         // keystore.add(key.hashCode());
         // else { do something with policyResult.deniedReason()? }
         // }
@@ -129,7 +129,6 @@ public void invalidate(K key) {
 
         // if (keystore.contains(key.hashCode()) {
         cache.remove(key);
-        //count--;
         // keystore.remove(key.hashCode());
         // }
     }
@@ -159,7 +158,6 @@ public Iterable<K> keys() {
     @Override
     public int count() {
         return (int) count.count();
-        //return (int) getTierStats().getMappings();
     }
 
     protected void countInc() {
@@ -183,35 +181,6 @@ public double getTimeMillisEWMA() {
         return getTimeMillisEWMA.getAverage();
     }
 
-    // these aren't really needed, ShardRequestCache handles it
-    // Also, it seems that ehcache doesn't have functioning statistics anyway!
-
-    /*public TierStatistics getTierStats() {
-        return statsService.getCacheStatistics(cacheAlias).getTierStatistics().get("Disk");
-    }
-
-    public long getHits() {
-        return getTierStats().getHits();
-    }
-
-    public long getMisses() {
-        return getTierStats().getMisses();
-    }
-
-    public long getWeightBytes() {
-        return getTierStats().getOccupiedByteSize();
-    }
-
-    public long getEvictions() {
-        return getTierStats().getEvictions();
-    }
-
-    public double getHitRatio() {
-        TierStatistics ts = getTierStats();
-        long hits = ts.getHits();
-        return hits / (hits + ts.getMisses());
-    }*/
-
     public boolean isPersistent() {
         return isPersistent;
     }
@@ -222,12 +191,4 @@ public void close() {
         cacheManager.removeCache(cacheAlias);
         cacheManager.close();
     }
-
-
-    // See https://stackoverflow.com/questions/45827753/listenerobject-not-found-in-imports-for-ehcache-3 for API reference
-    // it's not actually documented by ehcache :(
-    // This class is used to get the old value from mutating calls to the cache, and it uses those to create a RemovalNotification
-    // It also handles incrementing and decrementing the count for the disk tier, since ehcache's statistics functionality
-    // does not seem to work
-
 }
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 9a475be609a1a..80565f3fe127e 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -46,7 +46,6 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.ConcurrentCollections;
-import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.unit.ByteSizeValue;
 
@@ -107,8 +106,8 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
     private final TimeValue expire;
     // private final Cache<Key, BytesReference> cache;
 
-    //private final TieredCacheHandler<Key, BytesReference> tieredCacheHandler; // made public TieredCacheSpilloverStrategyHandler for testing
-    public final TieredCacheSpilloverStrategyHandler<Key, BytesReference> tieredCacheHandler;
+    //private final TieredCacheHandler<Key, BytesReference> tieredCacheHandler;
+    public final TieredCacheSpilloverStrategyHandler<Key, BytesReference> tieredCacheHandler; // Change this back after done debugging serialization issues
     IndicesRequestCache(Settings settings) {
         this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
         this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index 5657e3519f2ca..589c7dcfd9887 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -36,7 +36,7 @@ public class TieredCacheSpilloverStrategyHandler<K, V> implements TieredCacheHan
     private TieredCacheSpilloverStrategyHandler(
         OnHeapCachingTier<K, V> onHeapCachingTier,
         EhcacheDiskCachingTier<K, V> diskCachingTier,
-        // changed to EhcacheDiskCachingTier from CachingTier, to enable close() method, which is needed for tests. Implement close() in CachingTier or DiskCachingTier or something?
+        // changed to EhcacheDiskCachingTier from CachingTier, to enable close() method, which is needed for tests. Implement close() in CachingTier or DiskCachingTier?
         TieredCacheEventListener<K, V> tieredCacheEventListener
     ) {
         this.onHeapCachingTier = Objects.requireNonNull(onHeapCachingTier);
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 8cdf5635439af..14cca8ef60681 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -142,6 +142,9 @@ public void testBasicOperationsCache() throws Exception {
     }
 
     public void testAddDirectToEhcache() throws Exception {
+        // this test is for debugging serialization issues and can eventually be removed
+        // Put breakpoints at line 260 of AbstractOffHeapStore to catch serialization errors
+        // that would otherwise fail silently
         ShardRequestCache requestCacheStats = new ShardRequestCache();
         Settings.Builder settingsBuilder = Settings.builder();
         long heapSizeBytes = 1000;
@@ -162,7 +165,9 @@ public void testAddDirectToEhcache() throws Exception {
         IndicesRequestCache.Key key = new IndicesRequestCache.Key(entity, reader.getReaderCacheHelper().getKey(), termBytes);
 
         TestBytesReference value = new TestBytesReference(124);
-        cache.tieredCacheHandler.getDiskCachingTier().cache.put(key, value);
+        cache.tieredCacheHandler.getDiskCachingTier().put(key, value);
+
+        System.out.println("Size: " + cache.tieredCacheHandler.getDiskCachingTier().count());
 
         IOUtils.close(reader, writer, dir, cache);
         cache.closeDiskTier();

From 7a7b41fa951c3312b21a4bf1aa3fa7f0be1f237a Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 16 Oct 2023 12:00:33 -0700
Subject: [PATCH 020/120] More cleanup

---
 .../java/org/opensearch/common/metrics/CounterMetric.java   | 1 +
 .../opensearch/index/cache/request/ShardRequestCache.java   | 3 ++-
 .../org/opensearch/indices/IndicesRequestCacheTests.java    | 6 ++++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java b/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
index cb181840406a5..33fc5e32e9c60 100644
--- a/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
+++ b/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.common.metrics;
 
+import java.io.Serializable;
 import java.util.concurrent.atomic.LongAdder;
 
 /**
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index f09d2504680c7..3194aee757fc4 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -37,6 +37,7 @@
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.indices.TierType;
 
+import java.io.Serializable;
 import java.util.EnumMap;
 
 /**
@@ -113,7 +114,7 @@ public void onRemoval(Accountable key, BytesReference value, boolean evicted, Ti
         statsHolder.get(tierType).totalMetric.dec(dec);
     }
 
-    static class StatsHolder {
+    static class StatsHolder implements Serializable {
 
         final CounterMetric evictionsMetric = new CounterMetric();
         final CounterMetric totalMetric = new CounterMetric();
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 14cca8ef60681..5636268d78eb1 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -67,6 +67,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.bytes.AbstractBytesReference;
+import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.index.shard.ShardId;
@@ -164,7 +165,8 @@ public void testAddDirectToEhcache() throws Exception {
         BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
         IndicesRequestCache.Key key = new IndicesRequestCache.Key(entity, reader.getReaderCacheHelper().getKey(), termBytes);
 
-        TestBytesReference value = new TestBytesReference(124);
+        //TestBytesReference value = new TestBytesReference(124);
+        BytesReference value = new BytesArray(new byte[]{0});
         cache.tieredCacheHandler.getDiskCachingTier().put(key, value);
 
         System.out.println("Size: " + cache.tieredCacheHandler.getDiskCachingTier().count());
@@ -707,7 +709,7 @@ public boolean isFragment() {
         }
     }
 
-    private class TestEntity extends AbstractIndexShardCacheEntity {
+    private class TestEntity extends AbstractIndexShardCacheEntity implements Serializable {
         private final AtomicBoolean standInForIndexShard;
         private final ShardRequestCache shardRequestCache;
 

From 990ba9cdb73fca678aca134e00cbfdaf959fc2c9 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 16 Oct 2023 16:25:06 -0700
Subject: [PATCH 021/120] Part 1 of modifying ehcache to take serializable Key

---
 .../core/common/bytes/BytesReference.java     |  2 +-
 .../org/opensearch/indices/CachingTier.java   |  8 ++-
 .../opensearch/indices/DiskCachingTier.java   |  5 +-
 .../indices/EhcacheDiskCachingTier.java       | 59 +++++++++++++------
 .../indices/EhcacheEventListener.java         | 31 ++++++----
 .../org/opensearch/indices/EhcacheKey.java    | 54 +++++++++++++++++
 .../TieredCacheSpilloverStrategyHandler.java  |  9 +--
 7 files changed, 129 insertions(+), 39 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/indices/EhcacheKey.java

diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
index 60613d4e2a997..e07aca89339b9 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
@@ -55,7 +55,7 @@
  * @opensearch.api
  */
 @PublicApi(since = "1.0.0")
-public interface BytesReference extends Comparable<BytesReference>, ToXContentFragment, Serializable { // another lie!
+public interface BytesReference extends Comparable<BytesReference>, ToXContentFragment {
 
     /**
      * Convert an {@link XContentBuilder} into a BytesReference. This method closes the builder,
diff --git a/server/src/main/java/org/opensearch/indices/CachingTier.java b/server/src/main/java/org/opensearch/indices/CachingTier.java
index 85596929cfd6b..8c0dc0936f9dc 100644
--- a/server/src/main/java/org/opensearch/indices/CachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/CachingTier.java
@@ -10,6 +10,8 @@
 
 import org.opensearch.common.cache.RemovalListener;
 
+import java.io.IOException;
+
 /**
  * asdsadssa
  * @param <K>
@@ -17,13 +19,13 @@
  */
 public interface CachingTier<K, V> {
 
-    V get(K key);
+    V get(K key) throws IOException;
 
-    void put(K key, V value);
+    void put(K key, V value) throws IOException;
 
     V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception;
 
-    void invalidate(K key);
+    void invalidate(K key) throws IOException;
 
     V compute(K key, TieredCacheLoader<K, V> loader) throws Exception;
 
diff --git a/server/src/main/java/org/opensearch/indices/DiskCachingTier.java b/server/src/main/java/org/opensearch/indices/DiskCachingTier.java
index efd9a459cd338..de88c44cc1e8f 100644
--- a/server/src/main/java/org/opensearch/indices/DiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/DiskCachingTier.java
@@ -9,5 +9,8 @@
 package org.opensearch.indices;
 
 public interface DiskCachingTier<K, V> extends CachingTier<K, V> {
-
+    /**
+     * Closes the disk tier.
+     */
+    void close();
 }
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 2142b8f3b0930..4a34fea847981 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -27,16 +27,24 @@
 import org.ehcache.Cache;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.metrics.CounterMetric;
-
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.Collections;
 
-public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalListener<K, V> {
+public class EhcacheDiskCachingTier<K extends Writeable, V> implements DiskCachingTier<K, V>, RemovalListener<K, V> {
 
     private final PersistentCacheManager cacheManager;
-    private final Cache<K, V> cache; // make private after debug
-
-    private final Class<K> keyType; // I think these are needed to pass to newCacheConfigurationBuilder
+    private final Cache<EhcacheKey, V> cache;
+    private final Class<K> keyType; // These are needed to pass to newCacheConfigurationBuilder
+    //private final Class<EhcacheKey<K>> ehcacheKeyType;
     private final Class<V> valueType;
     public final static String DISK_CACHE_FP = "disk_cache_tier"; // this should probably be defined somewhere else since we need to change security.policy based on its value
     private RemovalListener<K, V> removalListener;
@@ -48,17 +56,25 @@ public class EhcacheDiskCachingTier<K, V> implements CachingTier<K, V>, RemovalL
     private static final String cacheAlias = "diskTier";
     private final boolean isPersistent;
     private CounterMetric count; // number of entries in cache
-    private EhcacheEventListener listener;
+    private final EhcacheEventListener<K, V> listener;
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
     // private IndicesRequestCacheDiskTierPolicy policy;
 
-    public EhcacheDiskCachingTier(boolean isPersistent, long maxWeightInBytes, long maxKeystoreWeightInBytes, Class<K> keyType, Class<V> valueType) {
+    public EhcacheDiskCachingTier(
+        boolean isPersistent,
+        long maxWeightInBytes,
+        long maxKeystoreWeightInBytes,
+        Class<K> keyType,
+        //Class<EhcacheKey<K>> ehcacheKeyType,
+        Class<V> valueType
+    ) {
+        this.isPersistent = isPersistent;
         this.keyType = keyType;
+        //this.ehcacheKeyType = ehcacheKeyType;
         this.valueType = valueType;
-        this.isPersistent = isPersistent;
         this.count = new CounterMetric();
-        this.listener = new EhcacheEventListener<K, V>(this, this.count);
+        this.listener = new EhcacheEventListener<K, V>(this, this);
         statsService = new DefaultStatisticsService();
 
         // our EhcacheEventListener should receive events every time an entry is changed
@@ -80,10 +96,10 @@ public EhcacheDiskCachingTier(boolean isPersistent, long maxWeightInBytes, long
             .using(threadConfig)
             .with(CacheManagerBuilder.persistence(DISK_CACHE_FP))
             .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                keyType, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, isPersistent))
+                EhcacheKey.class, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, isPersistent))
                 .withService(listenerConfig)
             ).build(true);
-        this.cache = cacheManager.getCache(cacheAlias, keyType, valueType);
+        this.cache = cacheManager.getCache(cacheAlias, EhcacheKey.class, valueType);
         this.getTimeMillisEWMA = new ExponentiallyWeightedMovingAverage(GET_TIME_EWMA_ALPHA, 10);
 
         // this.keystore = new RBMIntKeyLookupStore((int) Math.pow(2, 28), maxKeystoreWeightInBytes);
@@ -92,12 +108,12 @@ public EhcacheDiskCachingTier(boolean isPersistent, long maxWeightInBytes, long
     }
 
     @Override
-    public V get(K key) {
+    public V get(K key) throws IOException {
         // I don't think we need to do the future stuff as the cache is threadsafe
 
         // if (keystore.contains(key.hashCode()) {
         long now = System.nanoTime();
-        V value = cache.get(key);
+        V value = cache.get(new EhcacheKey(key));
         double tookTimeMillis = ((double) (System.nanoTime() - now)) / 1000000;
         getTimeMillisEWMA.addValue(tookTimeMillis);
         return value;
@@ -106,12 +122,12 @@ public V get(K key) {
     }
 
     @Override
-    public void put(K key, V value) {
+    public void put(K key, V value) throws IOException {
         // No need to get old value, this is handled by EhcacheEventListener.
 
         // CheckDataResult policyResult = policy.checkData(value)
         // if (policyResult.isAccepted()) {
-        cache.put(key, value);
+        cache.put(new EhcacheKey(key), value);
         // keystore.add(key.hashCode());
         // else { do something with policyResult.deniedReason()? }
         // }
@@ -123,12 +139,12 @@ public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception
     }
 
     @Override
-    public void invalidate(K key) {
+    public void invalidate(K key) throws IOException {
         // keep keystore check to avoid unneeded disk seek
         // RemovalNotification is handled by EhcacheEventListener
 
         // if (keystore.contains(key.hashCode()) {
-        cache.remove(key);
+        cache.remove(new EhcacheKey(key));
         // keystore.remove(key.hashCode());
         // }
     }
@@ -185,6 +201,15 @@ public boolean isPersistent() {
         return isPersistent;
     }
 
+    public K convertEhcacheKeyToOriginal(EhcacheKey eKey) throws IOException {
+        BytesStreamInput is = new BytesStreamInput();
+        byte[] bytes = eKey.getBytes();
+        is.readBytes(bytes, 0, bytes.length);
+        // we somehow have to use the Reader thing in the Writeable interface
+        // otherwise its not generic
+    }
+
+    @Override
     public void close() {
         // Call this method after each test, otherwise the directory will stay locked and you won't be able to
         // initialize another IndicesRequestCache (for example in the next test that runs)
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
index c6d2790dda95f..6c355dba025e1 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
@@ -15,33 +15,34 @@
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.indices.EhcacheDiskCachingTier;
 
 // moved to another file for testing flexibility purposes
 
-public class EhcacheEventListener<K, V> implements CacheEventListener<K, V> { // make it private after debugging
+public class EhcacheEventListener<K extends Writeable, V> implements CacheEventListener<EhcacheKey, V> {
+    // Receives key-value pairs (BytesReference, BytesReference), but must transform into (Key, BytesReference)
+    // to send removal notifications
     private RemovalListener<K, V> removalListener;
-    private CounterMetric counter;
-    EhcacheEventListener(RemovalListener<K, V> removalListener, CounterMetric counter) {
+    private EhcacheDiskCachingTier tier;
+    EhcacheEventListener(RemovalListener<K, V> removalListener, EhcacheDiskCachingTier tier) {
         this.removalListener = removalListener;
-        this.counter = counter; // needed to handle count changes
+        this.tier = tier; // needed to handle count changes
     }
     @Override
-    public void onEvent(CacheEvent<? extends K, ? extends V> event) {
-        K key = event.getKey();
+    public void onEvent(CacheEvent<? extends EhcacheKey, ? extends V> event) {
+        EhcacheKey ehcacheKey = event.getKey();
         V oldValue = event.getOldValue();
         V newValue = event.getNewValue();
         EventType eventType = event.getType();
 
-        System.out.println("I am eventing!!");
-
         // handle changing count for the disk tier
         if (oldValue == null && newValue != null) {
-            counter.inc();
+            tier.countInc();
         } else if (oldValue != null && newValue == null) {
-            counter.dec();
-        } else {
-            int j; // breakpoint
+            tier.countDec();
         }
 
         // handle creating a RemovalReason, unless eventType is CREATED
@@ -63,6 +64,10 @@ public void onEvent(CacheEvent<? extends K, ? extends V> event) {
             default:
                 reason = null;
         }
-        removalListener.onRemoval(new RemovalNotification<K, V>(key, oldValue, reason));
+        try {
+            K key = tier.convertEhcacheKeyToOriginal(ehcacheKey);
+            removalListener.onRemoval(new RemovalNotification<K, V>(key, oldValue, reason));
+        } catch (Exception ignored) {}
+
     }
 }
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheKey.java b/server/src/main/java/org/opensearch/indices/EhcacheKey.java
new file mode 100644
index 0000000000000..b0a8d660b98f6
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/EhcacheKey.java
@@ -0,0 +1,54 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+import org.opensearch.core.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Arrays;
+
+public class EhcacheKey implements Serializable {
+    // the IndicesRequestCache.Key is not Serializable, but it is Writeable.
+    // We use the output stream's bytes in this wrapper class and implement the appropriate interfaces/methods.
+    // Unfortunately it's not possible to define this class as EhcacheKey<K> and use that as ehcache keys,
+    // because of type erasure. However, the only context EhcacheKey objects would be compared to one another
+    // is when they are used for the same cache, so they will always refer to the same K.
+    private byte[] bytes;
+
+    public EhcacheKey(Writeable key) throws IOException {
+        BytesStreamOutput os = new BytesStreamOutput(); // Should we pass in an expected size? If so, how big?
+        key.writeTo(os);
+        this.bytes = BytesReference.toBytes(os.bytes());
+    }
+
+    public byte[] getBytes() {
+        return this.bytes;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) {
+            return true;
+        }
+        if (!(o instanceof EhcacheKey)) {
+            return false;
+        }
+        EhcacheKey other = (EhcacheKey) o;
+        return Arrays.equals(this.bytes, other.bytes);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(this.bytes);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index 589c7dcfd9887..c240162423ee5 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -11,6 +11,7 @@
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.core.common.io.stream.Writeable;
 
 import java.util.Arrays;
 import java.util.List;
@@ -22,10 +23,10 @@
  * @param <K>
  * @param <V>
  */
-public class TieredCacheSpilloverStrategyHandler<K, V> implements TieredCacheHandler<K, V>, RemovalListener<K, V> {
+public class TieredCacheSpilloverStrategyHandler<K extends Writeable, V> implements TieredCacheHandler<K, V>, RemovalListener<K, V> {
 
     private final OnHeapCachingTier<K, V> onHeapCachingTier;
-    private final EhcacheDiskCachingTier<K, V> diskCachingTier; // changed for testing
+    private final DiskCachingTier<K, V> diskCachingTier; // changed for testing
     private final TieredCacheEventListener<K, V> tieredCacheEventListener;
 
     /**
@@ -109,7 +110,7 @@ public CachingTier<K, V> getOnHeapCachingTier() {
         return this.onHeapCachingTier;
     }
 
-    public EhcacheDiskCachingTier<K, V> getDiskCachingTier() { // change to CachingTier after debug
+    public DiskCachingTier<K, V> getDiskCachingTier() { // change to CachingTier after debug
         return this.diskCachingTier;
     }
 
@@ -147,7 +148,7 @@ public static class CacheValue<V> {
         }
     }
 
-    public static class Builder<K, V> {
+    public static class Builder<K extends Writeable, V> {
         private OnHeapCachingTier<K, V> onHeapCachingTier;
         private CachingTier<K, V> diskCachingTier;
         private TieredCacheEventListener<K, V> tieredCacheEventListener;

From d21af15875ffc5ac34c0fa33577f47509ee9eabf Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 17 Oct 2023 10:00:15 -0700
Subject: [PATCH 022/120] final attempts at doing it generally before learning
 that wasnt even needed

---
 .../core/common/bytes/BytesReference.java     |  2 +-
 .../org/opensearch/indices/CachingTier.java   |  6 ++--
 .../indices/EhcacheDiskCachingTier.java       | 34 +++++++++++++++----
 .../indices/EhcacheEventListener.java         |  2 +-
 .../indices/IndicesRequestCache.java          |  7 +++-
 .../indices/IndicesRequestCacheTests.java     | 11 +++---
 6 files changed, 45 insertions(+), 17 deletions(-)

diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
index e07aca89339b9..f349baeef3cd7 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
@@ -55,7 +55,7 @@
  * @opensearch.api
  */
 @PublicApi(since = "1.0.0")
-public interface BytesReference extends Comparable<BytesReference>, ToXContentFragment {
+public interface BytesReference extends Comparable<BytesReference>, ToXContentFragment, Serializable {
 
     /**
      * Convert an {@link XContentBuilder} into a BytesReference. This method closes the builder,
diff --git a/server/src/main/java/org/opensearch/indices/CachingTier.java b/server/src/main/java/org/opensearch/indices/CachingTier.java
index 8c0dc0936f9dc..6726167fe469d 100644
--- a/server/src/main/java/org/opensearch/indices/CachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/CachingTier.java
@@ -19,13 +19,13 @@
  */
 public interface CachingTier<K, V> {
 
-    V get(K key) throws IOException;
+    V get(K key);
 
-    void put(K key, V value) throws IOException;
+    void put(K key, V value);
 
     V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception;
 
-    void invalidate(K key) throws IOException;
+    void invalidate(K key);
 
     V compute(K key, TieredCacheLoader<K, V> loader) throws Exception;
 
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 4a34fea847981..3fe9873e1ab80 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -31,6 +31,7 @@
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.BytesStreamInput;
+import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
 
@@ -40,6 +41,7 @@
 import java.util.Collections;
 
 public class EhcacheDiskCachingTier<K extends Writeable, V> implements DiskCachingTier<K, V>, RemovalListener<K, V> {
+    // & Writeable.Reader<K> ?
 
     private final PersistentCacheManager cacheManager;
     private final Cache<EhcacheKey, V> cache;
@@ -108,12 +110,16 @@ public EhcacheDiskCachingTier(
     }
 
     @Override
-    public V get(K key) throws IOException {
+    public V get(K key)  {
         // I don't think we need to do the future stuff as the cache is threadsafe
 
         // if (keystore.contains(key.hashCode()) {
         long now = System.nanoTime();
-        V value = cache.get(new EhcacheKey(key));
+        V value = null;
+        try {
+            value = cache.get(new EhcacheKey(key));
+        } catch (IOException ignored) { // do smth with this later
+        }
         double tookTimeMillis = ((double) (System.nanoTime() - now)) / 1000000;
         getTimeMillisEWMA.addValue(tookTimeMillis);
         return value;
@@ -122,12 +128,15 @@ public V get(K key) throws IOException {
     }
 
     @Override
-    public void put(K key, V value) throws IOException {
+    public void put(K key, V value) {
         // No need to get old value, this is handled by EhcacheEventListener.
 
         // CheckDataResult policyResult = policy.checkData(value)
         // if (policyResult.isAccepted()) {
-        cache.put(new EhcacheKey(key), value);
+        try {
+            cache.put(new EhcacheKey(key), value);
+        } catch (IOException ignored) { // do smth with this later
+        }
         // keystore.add(key.hashCode());
         // else { do something with policyResult.deniedReason()? }
         // }
@@ -139,12 +148,15 @@ public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception
     }
 
     @Override
-    public void invalidate(K key) throws IOException {
+    public void invalidate(K key) {
         // keep keystore check to avoid unneeded disk seek
         // RemovalNotification is handled by EhcacheEventListener
 
         // if (keystore.contains(key.hashCode()) {
-        cache.remove(new EhcacheKey(key));
+        try {
+            cache.remove(new EhcacheKey(key));
+        } catch (IOException ignored) { // do smth with this later
+        }
         // keystore.remove(key.hashCode());
         // }
     }
@@ -207,6 +219,16 @@ public K convertEhcacheKeyToOriginal(EhcacheKey eKey) throws IOException {
         is.readBytes(bytes, 0, bytes.length);
         // we somehow have to use the Reader thing in the Writeable interface
         // otherwise its not generic
+        try {
+            K key = keyType.getDeclaredConstructor(new Class[]{StreamInput.class}).newInstance();
+            // This cannot be the proper way to do it
+            // but if it is, make K extend some interface guaranteeing it has such a constructor (which im sure already exists somewhere in OS)
+            return key;
+        } catch (Exception e) {
+            System.out.println("Was unable to reconstruct EhcacheKey into K");
+            e.printStackTrace();
+        }
+        return null;
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
index 6c355dba025e1..7ad8718222925 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
@@ -26,7 +26,7 @@ public class EhcacheEventListener<K extends Writeable, V> implements CacheEventL
     // Receives key-value pairs (BytesReference, BytesReference), but must transform into (Key, BytesReference)
     // to send removal notifications
     private RemovalListener<K, V> removalListener;
-    private EhcacheDiskCachingTier tier;
+    private EhcacheDiskCachingTier<K, V> tier;
     EhcacheEventListener(RemovalListener<K, V> removalListener, EhcacheDiskCachingTier tier) {
         this.removalListener = removalListener;
         this.tier = tier; // needed to handle count changes
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 6caa83768a5eb..e55ce6ae73bb8 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -294,7 +294,7 @@ interface CacheEntity extends Accountable, Writeable {
      *
      * @opensearch.internal
      */
-    class Key implements Accountable, Writeable {
+    class Key implements Accountable, Writeable, Writeable.Reader<Key> {
         private final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Key.class);
 
         public final CacheEntity entity; // use as identity equality
@@ -349,6 +349,11 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalString(readerCacheKeyUniqueId);
             out.writeBytesReference(value);
         }
+
+        @Override
+        public Key read(StreamInput in) throws IOException {
+            return new Key(in);
+        }
     }
 
     private class CleanupKey implements IndexReader.ClosedListener {
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 87d8495a136d4..6ef23c7bb0790 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -181,10 +181,10 @@ public void testAddDirectToEhcache() throws Exception {
         cache.closeDiskTier();
     }
 
-    public void testSimpleEhcache() throws Exception {
+    /*public void testSimpleEhcache() throws Exception {
         // for debug only, delete
-         CounterMetric count = new CounterMetric();
-         String cacheAlias = "dummy";
+        CounterMetric count = new CounterMetric();
+        String cacheAlias = "dummy";
 
         class DummyRemovalListener implements RemovalListener<Integer, String> {
             public DummyRemovalListener() { }
@@ -275,10 +275,11 @@ public void onRemoval(RemovalNotification<Integer, String> notification) {
         IndicesRequestCache.Key key = new IndicesRequestCache.Key(entity, reader.getReaderCacheHelper().getKey(), termBytes);*/
 
 
-        cacheManager.removeCache(cacheAlias);
+        /*cacheManager.removeCache(cacheAlias);
         cacheManager.close();
         //IOUtils.close(reader, writer, dir);
-    }
+
+    }*/
 
     public void testSpillover() throws Exception {
         // fill the on-heap cache until we spill over

From d152478ad7ad705b64be966b2e87f14009bfdcf0 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 17 Oct 2023 11:29:52 -0700
Subject: [PATCH 023/120] Changed Ehcache disk tier implementation to not be
 generic, fixed persistent cache locking issue

---
 .../indices/EhcacheDiskCachingTier.java       | 118 +++++++++++-------
 .../indices/EhcacheEventListener.java         |  26 ++--
 .../org/opensearch/indices/EhcacheKey.java    |   3 -
 .../indices/IndicesRequestCache.java          |  12 +-
 .../TieredCacheSpilloverStrategyHandler.java  |   8 +-
 5 files changed, 87 insertions(+), 80 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 3fe9873e1ab80..5676a4278a45f 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -22,6 +22,7 @@
 import org.ehcache.event.CacheEventListener;
 import org.ehcache.event.EventType;
 import org.ehcache.impl.config.executor.PooledExecutionServiceConfiguration;
+import org.opensearch.action.admin.indices.exists.indices.IndicesExistsAction;
 import org.opensearch.common.ExponentiallyWeightedMovingAverage;
 import org.opensearch.common.cache.RemovalListener;
 import org.ehcache.Cache;
@@ -40,17 +41,16 @@
 import java.io.OutputStream;
 import java.util.Collections;
 
-public class EhcacheDiskCachingTier<K extends Writeable, V> implements DiskCachingTier<K, V>, RemovalListener<K, V> {
+public class EhcacheDiskCachingTier implements DiskCachingTier<IndicesRequestCache.Key, BytesReference>, RemovalListener<IndicesRequestCache.Key, BytesReference> {
     // & Writeable.Reader<K> ?
 
-    private final PersistentCacheManager cacheManager;
-    private final Cache<EhcacheKey, V> cache;
-    private final Class<K> keyType; // These are needed to pass to newCacheConfigurationBuilder
+    public static PersistentCacheManager cacheManager;
+    private Cache<EhcacheKey, BytesReference> cache;
+    //private final Class<K> keyType; // These are needed to pass to newCacheConfigurationBuilder
     //private final Class<EhcacheKey<K>> ehcacheKeyType;
-    private final Class<V> valueType;
+    //private final Class<V> valueType;
     public final static String DISK_CACHE_FP = "disk_cache_tier"; // this should probably be defined somewhere else since we need to change security.policy based on its value
-    private RemovalListener<K, V> removalListener;
-    private StatisticsService statsService; // non functional
+    private RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener;
     private ExponentiallyWeightedMovingAverage getTimeMillisEWMA;
     private static final double GET_TIME_EWMA_ALPHA  = 0.3; // This is the value used elsewhere in OpenSearch
     private static final int MIN_WRITE_THREADS = 0;
@@ -58,7 +58,8 @@ public class EhcacheDiskCachingTier<K extends Writeable, V> implements DiskCachi
     private static final String cacheAlias = "diskTier";
     private final boolean isPersistent;
     private CounterMetric count; // number of entries in cache
-    private final EhcacheEventListener<K, V> listener;
+    private final EhcacheEventListener listener;
+    private final IndicesRequestCache indicesRequestCache; // only used to create new Keys
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
     // private IndicesRequestCacheDiskTierPolicy policy;
@@ -67,18 +68,44 @@ public EhcacheDiskCachingTier(
         boolean isPersistent,
         long maxWeightInBytes,
         long maxKeystoreWeightInBytes,
-        Class<K> keyType,
+        IndicesRequestCache indicesRequestCache
+        //Class<K> keyType,
         //Class<EhcacheKey<K>> ehcacheKeyType,
-        Class<V> valueType
+        //Class<V> valueType
     ) {
         this.isPersistent = isPersistent;
-        this.keyType = keyType;
+        //this.keyType = keyType;
         //this.ehcacheKeyType = ehcacheKeyType;
-        this.valueType = valueType;
+        //this.valueType = valueType;
         this.count = new CounterMetric();
-        this.listener = new EhcacheEventListener<K, V>(this, this);
-        statsService = new DefaultStatisticsService();
+        this.listener = new EhcacheEventListener(this, this);
+        this.indicesRequestCache = indicesRequestCache;
 
+        getManager();
+        getOrCreateCache(isPersistent, maxWeightInBytes);
+        this.getTimeMillisEWMA = new ExponentiallyWeightedMovingAverage(GET_TIME_EWMA_ALPHA, 10);
+
+        // this.keystore = new RBMIntKeyLookupStore((int) Math.pow(2, 28), maxKeystoreWeightInBytes);
+        // this.policies = new CacheTierPolicy[]{ new IndicesRequestCacheTookTimePolicy(settings, clusterSettings) };
+        // this.policy = new IndicesRequestCacheDiskTierPolicy(this.policies, true);
+    }
+
+    public static void getManager() {
+        // based on https://stackoverflow.com/questions/53756412/ehcache-org-ehcache-statetransitionexception-persistence-directory-already-lo
+        // resolving double-initialization issue when using OpenSearchSingleNodeTestCase
+        if (cacheManager == null) {
+            PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
+                .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
+                .build();
+
+            cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
+                .using(threadConfig)
+                .with(CacheManagerBuilder.persistence(DISK_CACHE_FP)
+                ).build(true);
+        }
+    }
+
+    private void getOrCreateCache(boolean isPersistent, long maxWeightInBytes) {
         // our EhcacheEventListener should receive events every time an entry is changed
         CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
             .newEventListenerConfiguration(listener,
@@ -87,35 +114,27 @@ public EhcacheDiskCachingTier(
                 EventType.REMOVED,
                 EventType.UPDATED,
                 EventType.CREATED)
-            .ordered().asynchronous(); // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
-
-        PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
-            .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
-            .build();
-
-        this.cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
-            .using(statsService) // https://stackoverflow.com/questions/40453859/how-to-get-ehcache-3-1-statistics
-            .using(threadConfig)
-            .with(CacheManagerBuilder.persistence(DISK_CACHE_FP))
-            .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                EhcacheKey.class, valueType, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, isPersistent))
-                .withService(listenerConfig)
-            ).build(true);
-        this.cache = cacheManager.getCache(cacheAlias, EhcacheKey.class, valueType);
-        this.getTimeMillisEWMA = new ExponentiallyWeightedMovingAverage(GET_TIME_EWMA_ALPHA, 10);
+            .ordered().asynchronous();
+        // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
 
-        // this.keystore = new RBMIntKeyLookupStore((int) Math.pow(2, 28), maxKeystoreWeightInBytes);
-        // this.policies = new CacheTierPolicy[]{ new IndicesRequestCacheTookTimePolicy(settings, clusterSettings) };
-        // this.policy = new IndicesRequestCacheDiskTierPolicy(this.policies, true);
+        try {
+            cache = cacheManager.createCache(cacheAlias,
+                CacheConfigurationBuilder.newCacheConfigurationBuilder(
+                        EhcacheKey.class, BytesReference.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, isPersistent))
+                    .withService(listenerConfig));
+        } catch (IllegalArgumentException e) {
+            // Thrown when the cache already exists, which may happen in test cases
+            cache = cacheManager.getCache(cacheAlias, EhcacheKey.class, BytesReference.class);
+        }
     }
 
     @Override
-    public V get(K key)  {
+    public BytesReference get(IndicesRequestCache.Key key)  {
         // I don't think we need to do the future stuff as the cache is threadsafe
 
         // if (keystore.contains(key.hashCode()) {
         long now = System.nanoTime();
-        V value = null;
+        BytesReference value = null;
         try {
             value = cache.get(new EhcacheKey(key));
         } catch (IOException ignored) { // do smth with this later
@@ -128,7 +147,7 @@ public V get(K key)  {
     }
 
     @Override
-    public void put(K key, V value) {
+    public void put(IndicesRequestCache.Key key, BytesReference value) {
         // No need to get old value, this is handled by EhcacheEventListener.
 
         // CheckDataResult policyResult = policy.checkData(value)
@@ -143,12 +162,12 @@ public void put(K key, V value) {
     }
 
     @Override
-    public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception {
+    public BytesReference computeIfAbsent(IndicesRequestCache.Key key, TieredCacheLoader<IndicesRequestCache.Key, BytesReference> loader) throws Exception {
         return null; // should not need to fill out, Cache.computeIfAbsent is always used
     }
 
     @Override
-    public void invalidate(K key) {
+    public void invalidate(IndicesRequestCache.Key key) {
         // keep keystore check to avoid unneeded disk seek
         // RemovalNotification is handled by EhcacheEventListener
 
@@ -162,12 +181,12 @@ public void invalidate(K key) {
     }
 
     @Override
-    public V compute(K key, TieredCacheLoader<K, V> loader) throws Exception {
+    public BytesReference compute(IndicesRequestCache.Key key, TieredCacheLoader<IndicesRequestCache.Key, BytesReference> loader) throws Exception {
         return null; // should not need to fill out, Cache.compute is always used
     }
 
     @Override
-    public void setRemovalListener(RemovalListener<K, V> removalListener) {
+    public void setRemovalListener(RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener) {
         this.removalListener = removalListener; // this is passed the spillover strategy, same as on-heap
     }
 
@@ -177,7 +196,7 @@ public void invalidateAll() {
     }
 
     @Override
-    public Iterable<K> keys() {
+    public Iterable<IndicesRequestCache.Key> keys() {
         // ehcache doesn't provide a method like this, because it might be a huge number of keys that consume all
         // the memory. Do we want this method for disk tier?
         return Collections::emptyIterator;
@@ -201,7 +220,7 @@ public TierType getTierType() {
     }
 
     @Override
-    public void onRemoval(RemovalNotification<K, V> notification) {
+    public void onRemoval(RemovalNotification<IndicesRequestCache.Key, BytesReference> notification) {
         removalListener.onRemoval(notification);
     }
 
@@ -213,19 +232,16 @@ public boolean isPersistent() {
         return isPersistent;
     }
 
-    public K convertEhcacheKeyToOriginal(EhcacheKey eKey) throws IOException {
+    public IndicesRequestCache.Key convertEhcacheKeyToOriginal(EhcacheKey eKey) throws IOException {
         BytesStreamInput is = new BytesStreamInput();
         byte[] bytes = eKey.getBytes();
         is.readBytes(bytes, 0, bytes.length);
         // we somehow have to use the Reader thing in the Writeable interface
         // otherwise its not generic
         try {
-            K key = keyType.getDeclaredConstructor(new Class[]{StreamInput.class}).newInstance();
-            // This cannot be the proper way to do it
-            // but if it is, make K extend some interface guaranteeing it has such a constructor (which im sure already exists somewhere in OS)
-            return key;
+            return indicesRequestCache.new Key(is);
         } catch (Exception e) {
-            System.out.println("Was unable to reconstruct EhcacheKey into K");
+            System.out.println("Was unable to reconstruct EhcacheKey into Key");
             e.printStackTrace();
         }
         return null;
@@ -238,4 +254,10 @@ public void close() {
         cacheManager.removeCache(cacheAlias);
         cacheManager.close();
     }
+
+    public void destroy() throws Exception {
+        // Close the cache and delete any persistent data associated with it
+        // Might also be appropriate after standalone tests
+        cacheManager.destroy();
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
index 7ad8718222925..567daa4a266ce 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
@@ -14,28 +14,22 @@
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
-import org.opensearch.common.metrics.CounterMetric;
-import org.opensearch.core.common.io.stream.BytesStreamInput;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.indices.EhcacheDiskCachingTier;
+import org.opensearch.core.common.bytes.BytesReference;
 
-// moved to another file for testing flexibility purposes
-
-public class EhcacheEventListener<K extends Writeable, V> implements CacheEventListener<EhcacheKey, V> {
+public class EhcacheEventListener implements CacheEventListener<EhcacheKey, BytesReference> {
     // Receives key-value pairs (BytesReference, BytesReference), but must transform into (Key, BytesReference)
     // to send removal notifications
-    private RemovalListener<K, V> removalListener;
-    private EhcacheDiskCachingTier<K, V> tier;
-    EhcacheEventListener(RemovalListener<K, V> removalListener, EhcacheDiskCachingTier tier) {
+    private final RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener;
+    private final EhcacheDiskCachingTier tier;
+    EhcacheEventListener(RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener, EhcacheDiskCachingTier tier) {
         this.removalListener = removalListener;
         this.tier = tier; // needed to handle count changes
     }
     @Override
-    public void onEvent(CacheEvent<? extends EhcacheKey, ? extends V> event) {
+    public void onEvent(CacheEvent<? extends EhcacheKey, ? extends BytesReference> event) {
         EhcacheKey ehcacheKey = event.getKey();
-        V oldValue = event.getOldValue();
-        V newValue = event.getNewValue();
+        BytesReference oldValue = event.getOldValue();
+        BytesReference newValue = event.getNewValue();
         EventType eventType = event.getType();
 
         // handle changing count for the disk tier
@@ -65,8 +59,8 @@ public void onEvent(CacheEvent<? extends EhcacheKey, ? extends V> event) {
                 reason = null;
         }
         try {
-            K key = tier.convertEhcacheKeyToOriginal(ehcacheKey);
-            removalListener.onRemoval(new RemovalNotification<K, V>(key, oldValue, reason));
+            IndicesRequestCache.Key key = tier.convertEhcacheKeyToOriginal(ehcacheKey);
+            removalListener.onRemoval(new RemovalNotification<>(key, oldValue, reason));
         } catch (Exception ignored) {}
 
     }
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheKey.java b/server/src/main/java/org/opensearch/indices/EhcacheKey.java
index b0a8d660b98f6..43695f4bcc26f 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheKey.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheKey.java
@@ -20,9 +20,6 @@
 public class EhcacheKey implements Serializable {
     // the IndicesRequestCache.Key is not Serializable, but it is Writeable.
     // We use the output stream's bytes in this wrapper class and implement the appropriate interfaces/methods.
-    // Unfortunately it's not possible to define this class as EhcacheKey<K> and use that as ehcache keys,
-    // because of type erasure. However, the only context EhcacheKey objects would be compared to one another
-    // is when they are used for the same cache, so they will always refer to the same K.
     private byte[] bytes;
 
     public EhcacheKey(Writeable key) throws IOException {
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index e55ce6ae73bb8..a38e9d71fb982 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -54,7 +54,6 @@
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.io.Serializable;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -132,8 +131,8 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         int diskTierWeight = 100 * 1048576; // 100 MB, for testing only
 
         // changed to Integer for testing of bulk writes
-        EhcacheDiskCachingTier<Key, BytesReference> diskCachingTier;
-        diskCachingTier = new EhcacheDiskCachingTier<>(true, diskTierWeight, 0, Key.class, BytesReference.class);
+        EhcacheDiskCachingTier diskCachingTier;
+        diskCachingTier = new EhcacheDiskCachingTier(false, diskTierWeight, 0, this); // making non-persistent for now
         tieredCacheHandler = new TieredCacheSpilloverStrategyHandler.Builder<Key, BytesReference>().setOnHeapCachingTier(
             openSearchOnHeapCache
         ).setOnDiskCachingTier(diskCachingTier).setTieredCacheEventListener(this).build();
@@ -294,7 +293,7 @@ interface CacheEntity extends Accountable, Writeable {
      *
      * @opensearch.internal
      */
-    class Key implements Accountable, Writeable, Writeable.Reader<Key> {
+    class Key implements Accountable, Writeable {
         private final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Key.class);
 
         public final CacheEntity entity; // use as identity equality
@@ -349,11 +348,6 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalString(readerCacheKeyUniqueId);
             out.writeBytesReference(value);
         }
-
-        @Override
-        public Key read(StreamInput in) throws IOException {
-            return new Key(in);
-        }
     }
 
     private class CleanupKey implements IndexReader.ClosedListener {
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index c240162423ee5..203f72c8ff979 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -36,7 +36,7 @@ public class TieredCacheSpilloverStrategyHandler<K extends Writeable, V> impleme
 
     private TieredCacheSpilloverStrategyHandler(
         OnHeapCachingTier<K, V> onHeapCachingTier,
-        EhcacheDiskCachingTier<K, V> diskCachingTier,
+        DiskCachingTier<K, V> diskCachingTier,
         // changed to EhcacheDiskCachingTier from CachingTier, to enable close() method, which is needed for tests. Implement close() in CachingTier or DiskCachingTier?
         TieredCacheEventListener<K, V> tieredCacheEventListener
     ) {
@@ -150,7 +150,7 @@ public static class CacheValue<V> {
 
     public static class Builder<K extends Writeable, V> {
         private OnHeapCachingTier<K, V> onHeapCachingTier;
-        private CachingTier<K, V> diskCachingTier;
+        private DiskCachingTier<K, V> diskCachingTier;
         private TieredCacheEventListener<K, V> tieredCacheEventListener;
 
         public Builder() {}
@@ -160,7 +160,7 @@ public Builder<K, V> setOnHeapCachingTier(OnHeapCachingTier<K, V> onHeapCachingT
             return this;
         }
 
-        public Builder<K, V> setOnDiskCachingTier(CachingTier<K, V> diskCachingTier) {
+        public Builder<K, V> setOnDiskCachingTier(DiskCachingTier<K, V> diskCachingTier) {
             this.diskCachingTier = diskCachingTier;
             return this;
         }
@@ -173,7 +173,7 @@ public Builder<K, V> setTieredCacheEventListener(TieredCacheEventListener<K, V>
         public TieredCacheSpilloverStrategyHandler<K, V> build() {
             return new TieredCacheSpilloverStrategyHandler<K, V>(
                 this.onHeapCachingTier,
-                (EhcacheDiskCachingTier<K, V>) this.diskCachingTier, // not sure why it was yelling about this, it already is an EhcacheDiskCachingTier
+                this.diskCachingTier, // not sure why it was yelling about this, it already is an EhcacheDiskCachingTier
                 this.tieredCacheEventListener
             );
         }

From 2d200fcedd8326b45276a8e0a9f5f0eaeea48dba Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 17 Oct 2023 12:33:53 -0700
Subject: [PATCH 024/120] Fixed multiple disk tier instance issue, made
 spillover test pass

---
 .../common/metrics/CounterMetric.java         |   1 -
 .../indices/EhcacheDiskCachingTier.java       |  34 +++--
 .../indices/EhcacheEventListener.java         |   2 +-
 .../org/opensearch/indices/EhcacheKey.java    |   1 -
 .../TieredCacheSpilloverStrategyHandler.java  |  11 +-
 .../indices/IndicesRequestCacheTests.java     | 139 +++---------------
 6 files changed, 51 insertions(+), 137 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java b/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
index 33fc5e32e9c60..2e5eae5ceebe0 100644
--- a/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
+++ b/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
@@ -63,5 +63,4 @@ public void dec(long n) {
     public long count() {
         return counter.sum();
     }
-
 }
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 5676a4278a45f..190cfa1b9f7cc 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -9,40 +9,30 @@
 package org.opensearch.indices;
 
 import org.ehcache.PersistentCacheManager;
+import org.ehcache.config.CacheRuntimeConfiguration;
 import org.ehcache.config.builders.CacheConfigurationBuilder;
 import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
 import org.ehcache.config.builders.CacheManagerBuilder;
 import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
 import org.ehcache.config.builders.ResourcePoolsBuilder;
 import org.ehcache.config.units.MemoryUnit;
-import org.ehcache.core.internal.statistics.DefaultStatisticsService;
-import org.ehcache.core.spi.service.StatisticsService;
-import org.ehcache.core.statistics.TierStatistics;
-import org.ehcache.event.CacheEvent;
-import org.ehcache.event.CacheEventListener;
+import org.ehcache.event.EventFiring;
+import org.ehcache.event.EventOrdering;
 import org.ehcache.event.EventType;
 import org.ehcache.impl.config.executor.PooledExecutionServiceConfiguration;
-import org.opensearch.action.admin.indices.exists.indices.IndicesExistsAction;
 import org.opensearch.common.ExponentiallyWeightedMovingAverage;
 import org.opensearch.common.cache.RemovalListener;
 import org.ehcache.Cache;
 import org.opensearch.common.cache.RemovalNotification;
-import org.opensearch.common.cache.RemovalReason;
-import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.BytesStreamInput;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.common.io.stream.Writeable;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.util.Collections;
+import java.util.EnumSet;
 
 public class EhcacheDiskCachingTier implements DiskCachingTier<IndicesRequestCache.Key, BytesReference>, RemovalListener<IndicesRequestCache.Key, BytesReference> {
-    // & Writeable.Reader<K> ?
 
     public static PersistentCacheManager cacheManager;
     private Cache<EhcacheKey, BytesReference> cache;
@@ -124,7 +114,22 @@ private void getOrCreateCache(boolean isPersistent, long maxWeightInBytes) {
                     .withService(listenerConfig));
         } catch (IllegalArgumentException e) {
             // Thrown when the cache already exists, which may happen in test cases
+            // In this case the listener is configured to send messages to some other disk tier instance, which we don't want
+            // (it was set up unnecessarily by the test case)
+
+            // change config of existing cache to use this listener rather than the one instantiated by the test case
             cache = cacheManager.getCache(cacheAlias, EhcacheKey.class, BytesReference.class);
+            // cache.getRuntimeConfiguration().cacheConfigurationListenerList contains the old listener, but it's private
+            // and theres no method to clear it unless you have the actual listener object, so it has to stay i think
+
+            cache.getRuntimeConfiguration().registerCacheEventListener(listener, EventOrdering.ORDERED, EventFiring.ASYNCHRONOUS,
+                EnumSet.of(
+                    EventType.EVICTED,
+                    EventType.EXPIRED,
+                    EventType.REMOVED,
+                    EventType.UPDATED,
+                    EventType.CREATED));
+            int k = 1;
         }
     }
 
@@ -204,6 +209,7 @@ public Iterable<IndicesRequestCache.Key> keys() {
 
     @Override
     public int count() {
+        int j = 0;
         return (int) count.count();
     }
 
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
index 567daa4a266ce..e14102fd80ed5 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
@@ -17,7 +17,7 @@
 import org.opensearch.core.common.bytes.BytesReference;
 
 public class EhcacheEventListener implements CacheEventListener<EhcacheKey, BytesReference> {
-    // Receives key-value pairs (BytesReference, BytesReference), but must transform into (Key, BytesReference)
+    // Receives key-value pairs (EhcacheKey, BytesReference), but must transform into (Key, BytesReference)
     // to send removal notifications
     private final RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener;
     private final EhcacheDiskCachingTier tier;
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheKey.java b/server/src/main/java/org/opensearch/indices/EhcacheKey.java
index 43695f4bcc26f..f8fa87932d66f 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheKey.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheKey.java
@@ -10,7 +10,6 @@
 
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.io.stream.BytesStreamInput;
 import org.opensearch.core.common.io.stream.Writeable;
 
 import java.io.IOException;
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index 203f72c8ff979..6a4aa812cf010 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -91,6 +91,15 @@ public long count() {
         return totalCount;
     }
 
+    public long count(TierType tierType) {
+        for (CachingTier<K, V> cachingTier : cachingTierList) {
+            if (cachingTier.getTierType() == tierType) {
+                return cachingTier.count();
+            }
+        }
+        return -1L;
+    }
+
     @Override
     public void onRemoval(RemovalNotification<K, V> notification) {
         if (RemovalReason.EVICTED.equals(notification.getRemovalReason())) {
@@ -173,7 +182,7 @@ public Builder<K, V> setTieredCacheEventListener(TieredCacheEventListener<K, V>
         public TieredCacheSpilloverStrategyHandler<K, V> build() {
             return new TieredCacheSpilloverStrategyHandler<K, V>(
                 this.onHeapCachingTier,
-                this.diskCachingTier, // not sure why it was yelling about this, it already is an EhcacheDiskCachingTier
+                this.diskCachingTier,
                 this.tieredCacheEventListener
             );
         }
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 6ef23c7bb0790..35da7bac938e6 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -59,11 +59,8 @@
 import org.ehcache.event.EventType;
 import org.ehcache.impl.config.executor.PooledExecutionServiceConfiguration;
 import org.opensearch.common.CheckedSupplier;
-import org.opensearch.common.cache.RemovalListener;
-import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
-import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.bytes.AbstractBytesReference;
@@ -164,128 +161,29 @@ public void testAddDirectToEhcache() throws Exception {
         DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
         AtomicBoolean indexShard = new AtomicBoolean(true);
         TestEntity entity = new TestEntity(requestCacheStats, indexShard);
-        Loader loader = new Loader(reader, 0);
-        IndicesRequestCache.Key[] keys = new IndicesRequestCache.Key[9];
         TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
         BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
         String rKey = ((OpenSearchDirectoryReader) reader).getDelegatingCacheHelper().getDelegatingCacheKey().getId().toString();
         IndicesRequestCache.Key key = cache.new Key(entity, termBytes, rKey);
 
-        //TestBytesReference value = new TestBytesReference(124);
         BytesReference value = new BytesArray(new byte[]{0});
         cache.tieredCacheHandler.getDiskCachingTier().put(key, value);
 
-        System.out.println("Size: " + cache.tieredCacheHandler.getDiskCachingTier().count());
+        BytesReference res = cache.tieredCacheHandler.getDiskCachingTier().get(key);
+        assertEquals(value, res);
+        assertEquals(1, cache.tieredCacheHandler.count(TierType.DISK));
 
         IOUtils.close(reader, writer, dir, cache);
         cache.closeDiskTier();
     }
 
-    /*public void testSimpleEhcache() throws Exception {
-        // for debug only, delete
-        CounterMetric count = new CounterMetric();
-        String cacheAlias = "dummy";
-
-        class DummyRemovalListener implements RemovalListener<Integer, String> {
-            public DummyRemovalListener() { }
-            @Override
-            public void onRemoval(RemovalNotification<Integer, String> notification) {
-                System.out.println(":)");
-            }
-        }
-
-        CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
-            .newEventListenerConfiguration(new EhcacheEventListener<Integer, String>(new DummyRemovalListener(), count),
-                EventType.EVICTED,
-                EventType.EXPIRED,
-                EventType.REMOVED,
-                EventType.UPDATED,
-                EventType.CREATED)
-        .ordered().asynchronous(); // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
-
-        StatisticsService statsService = new DefaultStatisticsService();
-
-        PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
-            .defaultPool("default", 0, 4)
-            .build();
-
-        PersistentCacheManager cacheManager;
-
-        boolean doIntCache = false;
-
-        if (doIntCache) {
-            cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
-                .using(statsService) // https://stackoverflow.com/questions/40453859/how-to-get-ehcache-3-1-statistics
-                .using(threadConfig)
-                .with(CacheManagerBuilder.persistence(EhcacheDiskCachingTier.DISK_CACHE_FP))
-                .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                        Integer.class, String.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(10, MemoryUnit.MB, false))
-                    .withService(listenerConfig)
-                ).build(true);
-            Cache<Integer, String> integerCache = cacheManager.getCache(cacheAlias, Integer.class, String.class);
-
-            integerCache.put(0, "blorp");
-            System.out.println("Counter value = " + count.count());
-            String res = integerCache.get(0);
-            System.out.println("Got result " + res);
-            System.out.println("Counter value = " + count.count());
-        } else {
-            cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
-                .using(statsService) // https://stackoverflow.com/questions/40453859/how-to-get-ehcache-3-1-statistics
-                .using(threadConfig)
-                .with(CacheManagerBuilder.persistence(EhcacheDiskCachingTier.DISK_CACHE_FP))
-                .withCache(cacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                        DummySerializableKey.class, String.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(10, MemoryUnit.MB, false))
-                    .withService(listenerConfig)
-                ).build(true);
-            Cache<DummySerializableKey, String> cache = cacheManager.getCache(cacheAlias, DummySerializableKey.class, String.class);
-
-            DummySerializableKey key = new DummySerializableKey(Integer.valueOf(0), "blah");
-            cache.put(key, "blorp");
-            System.out.println("Counter value = " + count.count());
-            String res = cache.get(key);
-            System.out.println("Got result " + res);
-            System.out.println("Counter value = " + count.count());
-            TierStatistics ts = statsService.getCacheStatistics(cacheAlias).getTierStatistics().get("Disk");
-            System.out.println("self-reported count = " + ts.getMappings());
-            System.out.println("self-reported misses = " + ts.getMisses());
-            System.out.println("self-reported hits = " + ts.getHits());
-
-            List<Cache.Entry<DummySerializableKey, String>> foos = new ArrayList<>();
-            for(Cache.Entry<DummySerializableKey, String> entry : cache) {
-                foos.add(entry);
-            }
-            int j = 0;
-            j++;
-            System.out.println(j);
-        }
-
-        /*Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-        writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        AtomicBoolean indexShard = new AtomicBoolean(true);
-        ShardRequestCache requestCacheStats = new ShardRequestCache();
-        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
-        Loader loader = new Loader(reader, 0);
-        System.out.println("On-heap cache size at start = " + requestCacheStats.stats().getMemorySizeInBytes());
-        IndicesRequestCache.Key[] keys = new IndicesRequestCache.Key[9];
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        IndicesRequestCache.Key key = new IndicesRequestCache.Key(entity, reader.getReaderCacheHelper().getKey(), termBytes);*/
-
-
-        /*cacheManager.removeCache(cacheAlias);
-        cacheManager.close();
-        //IOUtils.close(reader, writer, dir);
-
-    }*/
-
     public void testSpillover() throws Exception {
         // fill the on-heap cache until we spill over
         ShardRequestCache requestCacheStats = new ShardRequestCache();
         Settings.Builder settingsBuilder = Settings.builder();
-        long heapSizeBytes = 1000; // each of these queries is 115 bytes, so we can fit 8 in the heap cache
+        long heapSizeBytes = 1000; // each of these queries is 131 bytes, so we can fit 7 in the heap cache
+        int heapKeySize = 131;
+        int maxNumInHeap = 1000 / heapKeySize;
         settingsBuilder.put("indices.requests.cache.size", new ByteSizeValue(heapSizeBytes));
         IndicesRequestCache cache = new IndicesRequestCache(settingsBuilder.build(), getInstanceFromNode(IndicesService.class));
 
@@ -298,8 +196,8 @@ public void testSpillover() throws Exception {
         TestEntity entity = new TestEntity(requestCacheStats, indexShard);
         Loader loader = new Loader(reader, 0);
         System.out.println("On-heap cache size at start = " + requestCacheStats.stats().getMemorySizeInBytes());
-        IndicesRequestCache.Key[] keys = new IndicesRequestCache.Key[9];
-        for (int i = 0; i < 9; i++) {
+        IndicesRequestCache.Key[] keys = new IndicesRequestCache.Key[maxNumInHeap + 1];
+        for (int i = 0; i < maxNumInHeap + 1; i++) {
             TermQueryBuilder termQuery = new TermQueryBuilder("id", String.valueOf(i));
             BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
             String rKey = ((OpenSearchDirectoryReader) reader).getDelegatingCacheHelper().getDelegatingCacheKey().getId().toString();
@@ -310,15 +208,18 @@ public void testSpillover() throws Exception {
         }
         // attempt to get value from disk cache, the first key should have been evicted
         BytesReference firstValue = cache.tieredCacheHandler.get(keys[0]);
-        System.out.println("Final on-heap cache size = " + requestCacheStats.stats().getMemorySizeInBytes()); // is correctly 920
-        //System.out.println("Final self-reported disk size = " + cache.tieredCacheHandler.getDiskWeightBytes()); // is 0, should be 115
-        System.out.println("On-heap tier evictions = " + requestCacheStats.stats().getEvictions()); // is correctly 1
-        System.out.println("Disk tier hits = " + requestCacheStats.stats(TierType.DISK).getHitCount()); // should be 1, is 0 bc keys not serializable
-        System.out.println("Disk tier misses = " + requestCacheStats.stats(TierType.DISK).getMissCount()); // should be 9, is 10 bc keys not serializable
-        //System.out.println("Disk tier self-reported misses = " + cache.tieredCacheHandler.getDiskCachingTier().getMisses()); // should be same as other one
-        System.out.println("On-heap tier hits = " + requestCacheStats.stats().getHitCount()); // is correctly 0
-        System.out.println("On-heap tier misses = " + requestCacheStats.stats().getMissCount()); // is correctly 10
-        System.out.println("Disk count = " + cache.tieredCacheHandler.getDiskCachingTier().count()); // should be 1, is 0
+
+        assertEquals(maxNumInHeap * heapKeySize, requestCacheStats.stats().getMemorySizeInBytes());
+        // TODO: disk weight bytes
+        assertEquals(1, requestCacheStats.stats().getEvictions());
+        assertEquals(1, requestCacheStats.stats(TierType.DISK).getHitCount());
+        assertEquals(maxNumInHeap + 1, requestCacheStats.stats(TierType.DISK).getMissCount());
+        assertEquals(0, requestCacheStats.stats().getHitCount());
+        assertEquals(maxNumInHeap + 2, requestCacheStats.stats().getMissCount());
+        assertEquals(maxNumInHeap, cache.tieredCacheHandler.count(TierType.ON_HEAP));
+        assertEquals(1, cache.tieredCacheHandler.count(TierType.DISK));
+
+        // more? 
         IOUtils.close(reader, writer, dir, cache);
         cache.closeDiskTier();
     }

From 8f65db48e689b2c3f8b7834a127ac2ad3c6f8064 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 17 Oct 2023 15:11:48 -0700
Subject: [PATCH 025/120] Removed persistence, cleaned up

---
 .../opensearch/indices/DiskCachingTier.java   |  6 ++
 .../indices/EhcacheDiskCachingTier.java       | 97 ++++++++-----------
 .../indices/EhcacheEventListener.java         |  1 -
 .../indices/IndicesRequestCache.java          |  4 +-
 .../indices/TieredCacheHandler.java           |  2 +
 .../TieredCacheSpilloverStrategyHandler.java  |  5 +
 .../indices/IndicesRequestCacheTests.java     | 68 ++++++++-----
 7 files changed, 99 insertions(+), 84 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/DiskCachingTier.java b/server/src/main/java/org/opensearch/indices/DiskCachingTier.java
index de88c44cc1e8f..b1b5b03ed13cd 100644
--- a/server/src/main/java/org/opensearch/indices/DiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/DiskCachingTier.java
@@ -13,4 +13,10 @@ public interface DiskCachingTier<K, V> extends CachingTier<K, V> {
      * Closes the disk tier.
      */
     void close();
+
+    /**
+     * Get the EWMA time in milliseconds for a get().
+     * @return
+     */
+    double getTimeMillisEWMA();
 }
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 190cfa1b9f7cc..ce0fc52c9a96e 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -9,7 +9,6 @@
 package org.opensearch.indices;
 
 import org.ehcache.PersistentCacheManager;
-import org.ehcache.config.CacheRuntimeConfiguration;
 import org.ehcache.config.builders.CacheConfigurationBuilder;
 import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
 import org.ehcache.config.builders.CacheManagerBuilder;
@@ -36,17 +35,13 @@ public class EhcacheDiskCachingTier implements DiskCachingTier<IndicesRequestCac
 
     public static PersistentCacheManager cacheManager;
     private Cache<EhcacheKey, BytesReference> cache;
-    //private final Class<K> keyType; // These are needed to pass to newCacheConfigurationBuilder
-    //private final Class<EhcacheKey<K>> ehcacheKeyType;
-    //private final Class<V> valueType;
-    public final static String DISK_CACHE_FP = "disk_cache_tier"; // this should probably be defined somewhere else since we need to change security.policy based on its value
+    public final static String DISK_CACHE_FP = "disk_cache_tier"; // Placeholder. this should probably be defined somewhere else, since we need to change security.policy based on its value
     private RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener;
     private ExponentiallyWeightedMovingAverage getTimeMillisEWMA;
     private static final double GET_TIME_EWMA_ALPHA  = 0.3; // This is the value used elsewhere in OpenSearch
     private static final int MIN_WRITE_THREADS = 0;
     private static final int MAX_WRITE_THREADS = 4; // Max number of threads for the PooledExecutionService which handles writes
     private static final String cacheAlias = "diskTier";
-    private final boolean isPersistent;
     private CounterMetric count; // number of entries in cache
     private final EhcacheEventListener listener;
     private final IndicesRequestCache indicesRequestCache; // only used to create new Keys
@@ -55,24 +50,23 @@ public class EhcacheDiskCachingTier implements DiskCachingTier<IndicesRequestCac
     // private IndicesRequestCacheDiskTierPolicy policy;
 
     public EhcacheDiskCachingTier(
-        boolean isPersistent,
         long maxWeightInBytes,
         long maxKeystoreWeightInBytes,
         IndicesRequestCache indicesRequestCache
-        //Class<K> keyType,
-        //Class<EhcacheKey<K>> ehcacheKeyType,
-        //Class<V> valueType
     ) {
-        this.isPersistent = isPersistent;
-        //this.keyType = keyType;
-        //this.ehcacheKeyType = ehcacheKeyType;
-        //this.valueType = valueType;
         this.count = new CounterMetric();
         this.listener = new EhcacheEventListener(this, this);
         this.indicesRequestCache = indicesRequestCache;
 
         getManager();
-        getOrCreateCache(isPersistent, maxWeightInBytes);
+        try {
+            cacheManager.destroyCache(cacheAlias);
+        } catch (Exception e) {
+            System.out.println("Unable to destroy cache!!");
+            e.printStackTrace();
+            // do actual logging later
+        }
+        createCache(maxWeightInBytes);
         this.getTimeMillisEWMA = new ExponentiallyWeightedMovingAverage(GET_TIME_EWMA_ALPHA, 10);
 
         // this.keystore = new RBMIntKeyLookupStore((int) Math.pow(2, 28), maxKeystoreWeightInBytes);
@@ -83,19 +77,31 @@ public EhcacheDiskCachingTier(
     public static void getManager() {
         // based on https://stackoverflow.com/questions/53756412/ehcache-org-ehcache-statetransitionexception-persistence-directory-already-lo
         // resolving double-initialization issue when using OpenSearchSingleNodeTestCase
-        if (cacheManager == null) {
-            PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
-                .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
-                .build();
-
-            cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
-                .using(threadConfig)
-                .with(CacheManagerBuilder.persistence(DISK_CACHE_FP)
-                ).build(true);
+        if (cacheManager != null) {
+            try {
+                try {
+                    cacheManager.close();
+                } catch (IllegalStateException e) {
+                    System.out.println("Cache was uninitialized, skipping close() and moving to destroy()");
+                }
+                cacheManager.destroy();
+            } catch (Exception e) {
+                System.out.println("Was unable to destroy cache manager");
+                e.printStackTrace();
+                // actual logging later
+            }
         }
+        PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
+            .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
+            .build();
+
+        cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
+            .using(threadConfig)
+            .with(CacheManagerBuilder.persistence(DISK_CACHE_FP)
+            ).build(true);
     }
 
-    private void getOrCreateCache(boolean isPersistent, long maxWeightInBytes) {
+    private void createCache(long maxWeightInBytes) {
         // our EhcacheEventListener should receive events every time an entry is changed
         CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
             .newEventListenerConfiguration(listener,
@@ -107,30 +113,10 @@ private void getOrCreateCache(boolean isPersistent, long maxWeightInBytes) {
             .ordered().asynchronous();
         // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
 
-        try {
-            cache = cacheManager.createCache(cacheAlias,
-                CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                        EhcacheKey.class, BytesReference.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, isPersistent))
-                    .withService(listenerConfig));
-        } catch (IllegalArgumentException e) {
-            // Thrown when the cache already exists, which may happen in test cases
-            // In this case the listener is configured to send messages to some other disk tier instance, which we don't want
-            // (it was set up unnecessarily by the test case)
-
-            // change config of existing cache to use this listener rather than the one instantiated by the test case
-            cache = cacheManager.getCache(cacheAlias, EhcacheKey.class, BytesReference.class);
-            // cache.getRuntimeConfiguration().cacheConfigurationListenerList contains the old listener, but it's private
-            // and theres no method to clear it unless you have the actual listener object, so it has to stay i think
-
-            cache.getRuntimeConfiguration().registerCacheEventListener(listener, EventOrdering.ORDERED, EventFiring.ASYNCHRONOUS,
-                EnumSet.of(
-                    EventType.EVICTED,
-                    EventType.EXPIRED,
-                    EventType.REMOVED,
-                    EventType.UPDATED,
-                    EventType.CREATED));
-            int k = 1;
-        }
+        cache = cacheManager.createCache(cacheAlias,
+            CacheConfigurationBuilder.newCacheConfigurationBuilder(
+                    EhcacheKey.class, BytesReference.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, false))
+                .withService(listenerConfig));
     }
 
     @Override
@@ -209,7 +195,6 @@ public Iterable<IndicesRequestCache.Key> keys() {
 
     @Override
     public int count() {
-        int j = 0;
         return (int) count.count();
     }
 
@@ -230,33 +215,28 @@ public void onRemoval(RemovalNotification<IndicesRequestCache.Key, BytesReferenc
         removalListener.onRemoval(notification);
     }
 
+    @Override
     public double getTimeMillisEWMA() {
         return getTimeMillisEWMA.getAverage();
     }
 
-    public boolean isPersistent() {
-        return isPersistent;
-    }
-
     public IndicesRequestCache.Key convertEhcacheKeyToOriginal(EhcacheKey eKey) throws IOException {
         BytesStreamInput is = new BytesStreamInput();
         byte[] bytes = eKey.getBytes();
         is.readBytes(bytes, 0, bytes.length);
-        // we somehow have to use the Reader thing in the Writeable interface
-        // otherwise its not generic
         try {
             return indicesRequestCache.new Key(is);
         } catch (Exception e) {
             System.out.println("Was unable to reconstruct EhcacheKey into Key");
             e.printStackTrace();
+            // actual logging later
         }
         return null;
     }
 
     @Override
     public void close() {
-        // Call this method after each test, otherwise the directory will stay locked and you won't be able to
-        // initialize another IndicesRequestCache (for example in the next test that runs)
+        // Should be called after each test
         cacheManager.removeCache(cacheAlias);
         cacheManager.close();
     }
@@ -264,6 +244,7 @@ public void close() {
     public void destroy() throws Exception {
         // Close the cache and delete any persistent data associated with it
         // Might also be appropriate after standalone tests
+        cacheManager.close();
         cacheManager.destroy();
     }
 }
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
index e14102fd80ed5..27872269ba77b 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
@@ -62,6 +62,5 @@ public void onEvent(CacheEvent<? extends EhcacheKey, ? extends BytesReference> e
             IndicesRequestCache.Key key = tier.convertEhcacheKeyToOriginal(ehcacheKey);
             removalListener.onRemoval(new RemovalNotification<>(key, oldValue, reason));
         } catch (Exception ignored) {}
-
     }
 }
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index a38e9d71fb982..16aed8830cf84 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -129,10 +129,8 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         ).setMaximumWeight(sizeInBytes).setExpireAfterAccess(expire).build();
 
         int diskTierWeight = 100 * 1048576; // 100 MB, for testing only
-
-        // changed to Integer for testing of bulk writes
         EhcacheDiskCachingTier diskCachingTier;
-        diskCachingTier = new EhcacheDiskCachingTier(false, diskTierWeight, 0, this); // making non-persistent for now
+        diskCachingTier = new EhcacheDiskCachingTier(diskTierWeight, 0, this);
         tieredCacheHandler = new TieredCacheSpilloverStrategyHandler.Builder<Key, BytesReference>().setOnHeapCachingTier(
             openSearchOnHeapCache
         ).setOnDiskCachingTier(diskCachingTier).setTieredCacheEventListener(this).build();
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
index 0a70375790191..4816f94f7d619 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
@@ -23,4 +23,6 @@ public interface TieredCacheHandler<K, V> {
     CachingTier<K, V> getOnHeapCachingTier();
 
     void closeDiskTier();
+
+    double diskGetTimeMillisEWMA();
 }
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index 6a4aa812cf010..caa4b108946ac 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -147,6 +147,11 @@ public void closeDiskTier() {
         diskCachingTier.close();
     }
 
+    @Override
+    public double diskGetTimeMillisEWMA() {
+        return diskCachingTier.getTimeMillisEWMA();
+    }
+
     public static class CacheValue<V> {
         V value;
         TierType source;
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 35da7bac938e6..2bd2bd53a787a 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -45,19 +45,6 @@
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
-import org.ehcache.Cache;
-import org.ehcache.PersistentCacheManager;
-import org.ehcache.config.builders.CacheConfigurationBuilder;
-import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
-import org.ehcache.config.builders.CacheManagerBuilder;
-import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
-import org.ehcache.config.builders.ResourcePoolsBuilder;
-import org.ehcache.config.units.MemoryUnit;
-import org.ehcache.core.internal.statistics.DefaultStatisticsService;
-import org.ehcache.core.spi.service.StatisticsService;
-import org.ehcache.core.statistics.TierStatistics;
-import org.ehcache.event.EventType;
-import org.ehcache.impl.config.executor.PooledExecutionServiceConfiguration;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
@@ -80,9 +67,7 @@
 
 import java.io.IOException;
 import java.io.Serializable;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -196,18 +181,17 @@ public void testSpillover() throws Exception {
         TestEntity entity = new TestEntity(requestCacheStats, indexShard);
         Loader loader = new Loader(reader, 0);
         System.out.println("On-heap cache size at start = " + requestCacheStats.stats().getMemorySizeInBytes());
-        IndicesRequestCache.Key[] keys = new IndicesRequestCache.Key[maxNumInHeap + 1];
+        BytesReference[] termBytesArr = new BytesReference[maxNumInHeap + 1];
+
         for (int i = 0; i < maxNumInHeap + 1; i++) {
             TermQueryBuilder termQuery = new TermQueryBuilder("id", String.valueOf(i));
             BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
             String rKey = ((OpenSearchDirectoryReader) reader).getDelegatingCacheHelper().getDelegatingCacheKey().getId().toString();
-            keys[i] = cache.new Key(entity, termBytes, rKey);
+            termBytesArr[i] = termBytes;
             BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
-            System.out.println("On-heap cache size after " + (i+1) + " queries = " + requestCacheStats.stats().getMemorySizeInBytes());
-            System.out.println("Disk cache size after " + (i+1) + " queries = " + requestCacheStats.stats(TierType.DISK).getMemorySizeInBytes());
         }
-        // attempt to get value from disk cache, the first key should have been evicted
-        BytesReference firstValue = cache.tieredCacheHandler.get(keys[0]);
+        // get value from disk cache, the first key should have been evicted
+        BytesReference firstValue = cache.getOrCompute(entity, loader, reader, termBytesArr[0]);
 
         assertEquals(maxNumInHeap * heapKeySize, requestCacheStats.stats().getMemorySizeInBytes());
         // TODO: disk weight bytes
@@ -219,7 +203,47 @@ public void testSpillover() throws Exception {
         assertEquals(maxNumInHeap, cache.tieredCacheHandler.count(TierType.ON_HEAP));
         assertEquals(1, cache.tieredCacheHandler.count(TierType.DISK));
 
-        // more? 
+        // get a value from heap cache, second key should still be there
+        BytesReference secondValue = cache.getOrCompute(entity, loader, reader, termBytesArr[1]);
+        // get the value on disk cache again
+        BytesReference firstValueAgain = cache.getOrCompute(entity, loader, reader, termBytesArr[0]);
+
+        assertEquals(1, requestCacheStats.stats().getEvictions());
+        assertEquals(2, requestCacheStats.stats(TierType.DISK).getHitCount());
+        assertEquals(maxNumInHeap + 1, requestCacheStats.stats(TierType.DISK).getMissCount());
+        assertEquals(1, requestCacheStats.stats().getHitCount());
+        assertEquals(maxNumInHeap + 3, requestCacheStats.stats().getMissCount());
+        assertEquals(maxNumInHeap, cache.tieredCacheHandler.count(TierType.ON_HEAP));
+        assertEquals(1, cache.tieredCacheHandler.count(TierType.DISK));
+
+        IOUtils.close(reader, writer, dir, cache);
+        cache.closeDiskTier();
+    }
+
+    public void testDiskGetTimeEWMA() throws Exception {
+        ShardRequestCache requestCacheStats = new ShardRequestCache();
+        Settings.Builder settingsBuilder = Settings.builder();
+        long heapSizeBytes = 0; // skip directly to disk cache
+        settingsBuilder.put("indices.requests.cache.size", new ByteSizeValue(heapSizeBytes));
+        IndicesRequestCache cache = new IndicesRequestCache(settingsBuilder.build(), getInstanceFromNode(IndicesService.class));
+
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+        writer.addDocument(newDoc(0, "foo"));
+        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        AtomicBoolean indexShard = new AtomicBoolean(true);
+
+        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+        Loader loader = new Loader(reader, 0);
+
+        for (int i = 0; i < 50; i++) {
+            TermQueryBuilder termQuery = new TermQueryBuilder("id", String.valueOf(i));
+            BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+            BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
+            // on my machine get time EWMA converges to ~0.025 ms, but it does have an SSD
+            assertTrue(cache.tieredCacheHandler.diskGetTimeMillisEWMA() > 0);
+        }
+
         IOUtils.close(reader, writer, dir, cache);
         cache.closeDiskTier();
     }

From 7155afd7f651288a3416967b3087d0ff835f0353 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 17 Oct 2023 16:16:35 -0700
Subject: [PATCH 026/120] cleanup

---
 .../java/org/opensearch/indices/EhcacheDiskCachingTier.java    | 3 ---
 .../indices/TieredCacheSpilloverStrategyHandler.java           | 3 +--
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index ce0fc52c9a96e..2469fd5934ea0 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -15,8 +15,6 @@
 import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
 import org.ehcache.config.builders.ResourcePoolsBuilder;
 import org.ehcache.config.units.MemoryUnit;
-import org.ehcache.event.EventFiring;
-import org.ehcache.event.EventOrdering;
 import org.ehcache.event.EventType;
 import org.ehcache.impl.config.executor.PooledExecutionServiceConfiguration;
 import org.opensearch.common.ExponentiallyWeightedMovingAverage;
@@ -29,7 +27,6 @@
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.EnumSet;
 
 public class EhcacheDiskCachingTier implements DiskCachingTier<IndicesRequestCache.Key, BytesReference>, RemovalListener<IndicesRequestCache.Key, BytesReference> {
 
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index caa4b108946ac..db9d4ffe41447 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -26,7 +26,7 @@
 public class TieredCacheSpilloverStrategyHandler<K extends Writeable, V> implements TieredCacheHandler<K, V>, RemovalListener<K, V> {
 
     private final OnHeapCachingTier<K, V> onHeapCachingTier;
-    private final DiskCachingTier<K, V> diskCachingTier; // changed for testing
+    private final DiskCachingTier<K, V> diskCachingTier;
     private final TieredCacheEventListener<K, V> tieredCacheEventListener;
 
     /**
@@ -37,7 +37,6 @@ public class TieredCacheSpilloverStrategyHandler<K extends Writeable, V> impleme
     private TieredCacheSpilloverStrategyHandler(
         OnHeapCachingTier<K, V> onHeapCachingTier,
         DiskCachingTier<K, V> diskCachingTier,
-        // changed to EhcacheDiskCachingTier from CachingTier, to enable close() method, which is needed for tests. Implement close() in CachingTier or DiskCachingTier?
         TieredCacheEventListener<K, V> tieredCacheEventListener
     ) {
         this.onHeapCachingTier = Objects.requireNonNull(onHeapCachingTier);

From c760594789139a686cefc3e51269c1e40ff4fef6 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 18 Oct 2023 14:37:07 -0700
Subject: [PATCH 027/120] added test entries metric for on heap

---
 .../index/cache/request/RequestCacheStats.java      | 13 ++++++++++++-
 .../index/cache/request/ShardRequestCache.java      | 11 +++++++++--
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index 24f68899c2ac7..5aa9f57b64032 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -52,6 +52,7 @@ public class RequestCacheStats implements Writeable, ToXContentFragment {
     private long evictions;
     private long hitCount;
     private long missCount;
+    private long entries; // number of entries in the cache
 
     public RequestCacheStats() {}
 
@@ -60,13 +61,15 @@ public RequestCacheStats(StreamInput in) throws IOException {
         evictions = in.readVLong();
         hitCount = in.readVLong();
         missCount = in.readVLong();
+        entries = in.readVLong();
     }
 
-    public RequestCacheStats(long memorySize, long evictions, long hitCount, long missCount) {
+    public RequestCacheStats(long memorySize, long evictions, long hitCount, long missCount, long entries) { //
         this.memorySize = memorySize;
         this.evictions = evictions;
         this.hitCount = hitCount;
         this.missCount = missCount;
+        this.entries = entries;
     }
 
     public void add(RequestCacheStats stats) {
@@ -74,6 +77,7 @@ public void add(RequestCacheStats stats) {
         this.evictions += stats.evictions;
         this.hitCount += stats.hitCount;
         this.missCount += stats.missCount;
+        this.entries += stats.entries;
     }
 
     public long getMemorySizeInBytes() {
@@ -96,12 +100,17 @@ public long getMissCount() {
         return this.missCount;
     }
 
+    public long getEntries() {
+        return this.entries;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(memorySize);
         out.writeVLong(evictions);
         out.writeVLong(hitCount);
         out.writeVLong(missCount);
+        out.writeVLong(entries);
     }
 
     @Override
@@ -111,6 +120,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(Fields.EVICTIONS, getEvictions());
         builder.field(Fields.HIT_COUNT, getHitCount());
         builder.field(Fields.MISS_COUNT, getMissCount());
+        builder.field(Fields.ENTRIES, getEntries());
         builder.endObject();
         return builder;
     }
@@ -127,5 +137,6 @@ static final class Fields {
         static final String EVICTIONS = "evictions";
         static final String HIT_COUNT = "hit_count";
         static final String MISS_COUNT = "miss_count";
+        static final String ENTRIES = "entries";
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index 3194aee757fc4..d43418d63a6a6 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -65,7 +65,8 @@ public RequestCacheStats stats(TierType tierType) {
             statsHolder.get(tierType).totalMetric.count(),
             statsHolder.get(tierType).evictionsMetric.count(),
             statsHolder.get(tierType).hitCount.count(),
-            statsHolder.get(tierType).missCount.count()
+            statsHolder.get(tierType).missCount.count(),
+            statsHolder.get(tierType).entries.count()
         );
     }
 
@@ -74,17 +75,20 @@ public RequestCacheStats overallStats() {
         long totalEvictions = 0;
         long totalHits = 0;
         long totalMisses = 0;
+        long totalEntries = 0;
         for (TierType tierType : TierType.values()) {
             totalSize += statsHolder.get(tierType).totalMetric.count();
             totalEvictions += statsHolder.get(tierType).evictionsMetric.count();
             totalHits += statsHolder.get(tierType).hitCount.count();
             totalMisses += statsHolder.get(tierType).missCount.count();
+            totalEntries += statsHolder.get(tierType).entries.count();
         }
         return new RequestCacheStats(
             totalSize,
             totalEvictions,
             totalHits,
-            totalMisses
+            totalMisses,
+            totalEntries
         );
     }
 
@@ -98,6 +102,7 @@ public void onMiss(TierType tierType) {
 
     public void onCached(Accountable key, BytesReference value, TierType tierType) {
         statsHolder.get(tierType).totalMetric.inc(key.ramBytesUsed() + value.ramBytesUsed());
+        statsHolder.get(tierType).entries.inc();
     }
 
     public void onRemoval(Accountable key, BytesReference value, boolean evicted, TierType tierType) {
@@ -112,6 +117,7 @@ public void onRemoval(Accountable key, BytesReference value, boolean evicted, Ti
             dec += value.ramBytesUsed();
         }
         statsHolder.get(tierType).totalMetric.dec(dec);
+        statsHolder.get(tierType).entries.dec();
     }
 
     static class StatsHolder implements Serializable {
@@ -120,5 +126,6 @@ static class StatsHolder implements Serializable {
         final CounterMetric totalMetric = new CounterMetric();
         final CounterMetric hitCount = new CounterMetric();
         final CounterMetric missCount = new CounterMetric();
+        final CounterMetric entries = new CounterMetric();
     }
 }

From 8510637aad3f44fbeba200a8e6565c588a0a98e8 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 18 Oct 2023 15:26:43 -0700
Subject: [PATCH 028/120] Fixed issue where multiple mock nodes on one JVM (in
 IT tests) would cause initialization errors in the cache manager

---
 .../indices/EhcacheDiskCachingTier.java       | 48 ++++++++++++++-----
 .../indices/IndicesRequestCache.java          |  2 +-
 .../opensearch/indices/IndicesService.java    |  4 ++
 3 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 2469fd5934ea0..01fe6d491a58a 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -21,18 +21,35 @@
 import org.opensearch.common.cache.RemovalListener;
 import org.ehcache.Cache;
 import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.BytesStreamInput;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
 
 public class EhcacheDiskCachingTier implements DiskCachingTier<IndicesRequestCache.Key, BytesReference>, RemovalListener<IndicesRequestCache.Key, BytesReference> {
 
-    public static PersistentCacheManager cacheManager;
+    public static HashMap<String, PersistentCacheManager> cacheManagers = new HashMap<>();
+    // Because of the way test cases are set up, each node may try to instantiate several disk caching tiers.
+    // Only one of them will be used, but there will be initialization errors when multiple cache managers try to
+    // use the same file path and create/get caches with the same alias. We resolve this with a static reference
+    // to a cache manager, which is populated if it is null and reused if it is not.
+    // (See https://stackoverflow.com/questions/53756412/ehcache-org-ehcache-statetransitionexception-persistence-directory-already-lo)
+    // To deal with IT cases, we need to create a manager per node, as otherwise nodes will try to reuse the same manager,
+    // so we get the correct cache manager by looking up the node ID in this map.
+    // I don't think any of this can happen in production, because nodes shouldn't share a JVM,
+    // and they should only instantiate their services once? But it's best to resolve it anyway.
+
+    private PersistentCacheManager cacheManager; // This is the manager this tier will actually use
     private Cache<EhcacheKey, BytesReference> cache;
-    public final static String DISK_CACHE_FP = "disk_cache_tier"; // Placeholder. this should probably be defined somewhere else, since we need to change security.policy based on its value
+    public final static String BASE_DISK_CACHE_FP = "disk_cache_tier";
+    // Placeholder. this should probably be defined somewhere else, since we need to change security.policy based on its value
+    // To accomodate test setups, where multiple nodes may exist on the same filesystem, we add the node ID to the end of this
+    // These will be subfolders of BASE_DISK_CACHE_FP
+    private final String diskCacheFP; // the one to use for this node
     private RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener;
     private ExponentiallyWeightedMovingAverage getTimeMillisEWMA;
     private static final double GET_TIME_EWMA_ALPHA  = 0.3; // This is the value used elsewhere in OpenSearch
@@ -42,6 +59,7 @@ public class EhcacheDiskCachingTier implements DiskCachingTier<IndicesRequestCac
     private CounterMetric count; // number of entries in cache
     private final EhcacheEventListener listener;
     private final IndicesRequestCache indicesRequestCache; // only used to create new Keys
+    private final String nodeId;
     // private RBMIntKeyLookupStore keystore;
     // private CacheTierPolicy[] policies;
     // private IndicesRequestCacheDiskTierPolicy policy;
@@ -49,11 +67,15 @@ public class EhcacheDiskCachingTier implements DiskCachingTier<IndicesRequestCac
     public EhcacheDiskCachingTier(
         long maxWeightInBytes,
         long maxKeystoreWeightInBytes,
-        IndicesRequestCache indicesRequestCache
+        IndicesRequestCache indicesRequestCache,
+        String nodeId
     ) {
         this.count = new CounterMetric();
         this.listener = new EhcacheEventListener(this, this);
         this.indicesRequestCache = indicesRequestCache;
+        this.nodeId = nodeId;
+        this.diskCacheFP = PathUtils.get(BASE_DISK_CACHE_FP, nodeId).toString();
+        // I know this function warns it shouldn't often be used, we can fix it to use the roots once we pick a final FP
 
         getManager();
         try {
@@ -71,19 +93,20 @@ public EhcacheDiskCachingTier(
         // this.policy = new IndicesRequestCacheDiskTierPolicy(this.policies, true);
     }
 
-    public static void getManager() {
+    public void getManager() {
         // based on https://stackoverflow.com/questions/53756412/ehcache-org-ehcache-statetransitionexception-persistence-directory-already-lo
         // resolving double-initialization issue when using OpenSearchSingleNodeTestCase
-        if (cacheManager != null) {
+        PersistentCacheManager oldCacheManager = cacheManagers.get(nodeId);
+        if (oldCacheManager != null) {
             try {
                 try {
-                    cacheManager.close();
+                    oldCacheManager.close();
                 } catch (IllegalStateException e) {
                     System.out.println("Cache was uninitialized, skipping close() and moving to destroy()");
                 }
-                cacheManager.destroy();
+                oldCacheManager.destroy();
             } catch (Exception e) {
-                System.out.println("Was unable to destroy cache manager");
+                System.out.println("Was unable to destroy existing cache manager");
                 e.printStackTrace();
                 // actual logging later
             }
@@ -92,10 +115,13 @@ public static void getManager() {
             .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
             .build();
 
-        cacheManager = CacheManagerBuilder.newCacheManagerBuilder()
+        cacheManagers.put(nodeId,
+            CacheManagerBuilder.newCacheManagerBuilder()
             .using(threadConfig)
-            .with(CacheManagerBuilder.persistence(DISK_CACHE_FP)
-            ).build(true);
+            .with(CacheManagerBuilder.persistence(diskCacheFP)
+            ).build(true)
+        );
+        this.cacheManager = cacheManagers.get(nodeId);
     }
 
     private void createCache(long maxWeightInBytes) {
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 16aed8830cf84..d3084c8c2c231 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -130,7 +130,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
 
         int diskTierWeight = 100 * 1048576; // 100 MB, for testing only
         EhcacheDiskCachingTier diskCachingTier;
-        diskCachingTier = new EhcacheDiskCachingTier(diskTierWeight, 0, this);
+        diskCachingTier = new EhcacheDiskCachingTier(diskTierWeight, 0, this, indicesService.getNodeId());
         tieredCacheHandler = new TieredCacheSpilloverStrategyHandler.Builder<Key, BytesReference>().setOnHeapCachingTier(
             openSearchOnHeapCache
         ).setOnDiskCachingTier(diskCachingTier).setTieredCacheEventListener(this).build();
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index f5e71327b6e7b..4be7061deabf3 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -1939,6 +1939,10 @@ public boolean allPendingDanglingIndicesWritten() {
             || (danglingIndicesToWrite.isEmpty() && danglingIndicesThreadPoolExecutor.getActiveCount() == 0);
     }
 
+    public String getNodeId() {
+        return nodeEnv.nodeId();
+    }
+
     /**
      * Validates the cluster default index refresh interval.
      *

From 4680ea7eac5275dec812181517e281266f77109a Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 18 Oct 2023 16:07:03 -0700
Subject: [PATCH 029/120] Added check for new entries field in IT test, added
 permissions allowing the domain to run

---
 .../opensearch/indices/IndicesRequestCacheIT.java  | 14 ++++++++++++++
 .../opensearch/indices/EhcacheDiskCachingTier.java |  2 --
 .../org/opensearch/bootstrap/security.policy       |  2 ++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index a1815d9be2daf..36e327ce11b6a 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -663,6 +663,7 @@ public void testCacheWithInvalidation() throws Exception {
         assertSearchResponse(resp);
         // Should expect hit as here as refresh didn't happen
         assertCacheState(client, "index", 1, 1);
+        assertNumCacheEntries(client, "index", 1);
 
         // Explicit refresh would invalidate cache
         refresh();
@@ -671,6 +672,8 @@ public void testCacheWithInvalidation() throws Exception {
         assertSearchResponse(resp);
         // Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh)
         assertCacheState(client, "index", 1, 2);
+        assertNumCacheEntries(client, "index", 1); // Shouldn't it just be the most recent query, since the first one was invalidated? (prob invalidation isnt in yet)
+        // yeah - evictions = 0, its not in yet
     }
 
     private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
@@ -690,4 +693,15 @@ private static void assertCacheState(Client client, String index, long expectedH
 
     }
 
+    private static void assertNumCacheEntries(Client client, String index, long expectedEntries) {
+        RequestCacheStats requestCacheStats = client.admin()
+            .indices()
+            .prepareStats(index)
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+        assertEquals(expectedEntries, requestCacheStats.getEntries());
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 01fe6d491a58a..2967600ff612a 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -94,8 +94,6 @@ public EhcacheDiskCachingTier(
     }
 
     public void getManager() {
-        // based on https://stackoverflow.com/questions/53756412/ehcache-org-ehcache-statetransitionexception-persistence-directory-already-lo
-        // resolving double-initialization issue when using OpenSearchSingleNodeTestCase
         PersistentCacheManager oldCacheManager = cacheManagers.get(nodeId);
         if (oldCacheManager != null) {
             try {
diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy
index 1fbfbb323e3af..cc7d83e07c6a1 100644
--- a/server/src/main/resources/org/opensearch/bootstrap/security.policy
+++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy
@@ -192,6 +192,8 @@ grant {
   permission java.lang.RuntimePermission "createClassLoader";
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
   permission java.lang.RuntimePermission "getenv.*";
+  permission java.lang.RuntimePermission "accessDeclaredMembers";
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   permission java.io.FilePermission "disk_cache_tier", "read"; // change this to wherever we will put disk tier folder
   permission java.io.FilePermission "disk_cache_tier", "write";
   permission java.io.FilePermission "disk_cache_tier", "delete";

From c300d8bf8d0ad79fc9674bdfdcbedd7d691d41df Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Fri, 20 Oct 2023 15:51:38 -0700
Subject: [PATCH 030/120] Initial implementation for tiered node request cache
 stats

---
 .../cache/request/RequestCacheStats.java      | 120 ++++++++++++------
 .../cache/request/ShardRequestCache.java      |  46 +------
 .../index/cache/request/StatsHolder.java      |  99 +++++++++++++++
 .../java/org/opensearch/indices/TierType.java |  15 ++-
 .../indices/IndicesRequestCacheTests.java     |   8 +-
 5 files changed, 200 insertions(+), 88 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java

diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index 5aa9f57b64032..5480dd5b01240 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -38,8 +38,11 @@
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.indices.TierType;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * Request for the query cache statistics
@@ -48,79 +51,117 @@
  */
 public class RequestCacheStats implements Writeable, ToXContentFragment {
 
-    private long memorySize;
-    private long evictions;
-    private long hitCount;
-    private long missCount;
-    private long entries; // number of entries in the cache
-
-    public RequestCacheStats() {}
+    private Map<String, StatsHolder> map;
+    public RequestCacheStats() {
+        this.map = new HashMap<>();
+        for (TierType tierType : TierType.values()) {
+            map.put(tierType.getStringValue(), new StatsHolder());
+            // Every possible tier type must have counters, even if they are disabled. Then the counters report 0
+        }
+    }
 
     public RequestCacheStats(StreamInput in) throws IOException {
-        memorySize = in.readVLong();
-        evictions = in.readVLong();
-        hitCount = in.readVLong();
-        missCount = in.readVLong();
-        entries = in.readVLong();
+        // Any RequestCacheStats written to a stream should already have a counter for each possible tier type
+        this.map = in.readMap(StreamInput::readString, StatsHolder::new); // does it know to use the right constructor? does it rly need to be registered?
+    }
+
+    public RequestCacheStats(TierType tierType, StatsHolder statsHolder) {
+        // Create a RequestCacheStats object with only one tier's statistics populated
+        this();
+        map.put(tierType.getStringValue(), statsHolder);
     }
 
-    public RequestCacheStats(long memorySize, long evictions, long hitCount, long missCount, long entries) { //
-        this.memorySize = memorySize;
-        this.evictions = evictions;
-        this.hitCount = hitCount;
-        this.missCount = missCount;
-        this.entries = entries;
+    public RequestCacheStats(Map<TierType, StatsHolder> inputMap) {
+        // Create a RequestCacheStats with multiple tiers' statistics
+        this();
+        for (TierType tierType : inputMap.keySet()) {
+            map.put(tierType.getStringValue(), inputMap.get(tierType));
+        }
     }
 
+    // can prob eliminate some of these constructors
+
     public void add(RequestCacheStats stats) {
-        this.memorySize += stats.memorySize;
-        this.evictions += stats.evictions;
-        this.hitCount += stats.hitCount;
-        this.missCount += stats.missCount;
-        this.entries += stats.entries;
+        for (String tier : stats.map.keySet()) {
+            map.get(tier).add(stats.map.get(tier));
+        }
+    }
+
+    private StatsHolder getTierStats(TierType tierType) {
+        return map.get(tierType.getStringValue());
+    }
+
+    // should these take in strings bc thats whats done in the xcontent builder? seems wasteful
+    public long getMemorySizeInBytes(TierType tierType) {
+        return getTierStats(tierType).totalMetric.count();
+    }
+
+    public ByteSizeValue getMemorySize(TierType tierType) {
+        return new ByteSizeValue(getMemorySizeInBytes(tierType));
     }
 
+    public long getEvictions(TierType tierType) {
+        return getTierStats(tierType).evictionsMetric.count();
+    }
+
+    public long getHitCount(TierType tierType) {
+        return getTierStats(tierType).hitCount.count();
+    }
+
+    public long getMissCount(TierType tierType) {
+        return getTierStats(tierType).missCount.count();
+    }
+
+    public long getEntries(TierType tierType) {
+        return getTierStats(tierType).entries.count();
+    }
+
+    // By default, return on-heap stats if no tier is specified
+
     public long getMemorySizeInBytes() {
-        return this.memorySize;
+        return getMemorySizeInBytes(TierType.ON_HEAP);
     }
 
     public ByteSizeValue getMemorySize() {
-        return new ByteSizeValue(memorySize);
+        return getMemorySize(TierType.ON_HEAP);
     }
 
     public long getEvictions() {
-        return this.evictions;
+        return getEvictions(TierType.ON_HEAP);
     }
 
     public long getHitCount() {
-        return this.hitCount;
+        return getHitCount(TierType.ON_HEAP);
     }
 
     public long getMissCount() {
-        return this.missCount;
+        return getMissCount(TierType.ON_HEAP);
     }
 
     public long getEntries() {
-        return this.entries;
+        return getEntries(TierType.ON_HEAP);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVLong(memorySize);
-        out.writeVLong(evictions);
-        out.writeVLong(hitCount);
-        out.writeVLong(missCount);
-        out.writeVLong(entries);
+        out.writeMap(this.map, StreamOutput::writeString, (o, v) -> v.writeTo(o)); // ?
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.REQUEST_CACHE_STATS);
-        builder.humanReadableField(Fields.MEMORY_SIZE_IN_BYTES, Fields.MEMORY_SIZE, getMemorySize());
-        builder.field(Fields.EVICTIONS, getEvictions());
-        builder.field(Fields.HIT_COUNT, getHitCount());
-        builder.field(Fields.MISS_COUNT, getMissCount());
-        builder.field(Fields.ENTRIES, getEntries());
+        // write on-heap stats outside of tiers object
+        getTierStats(TierType.ON_HEAP).toXContent(builder, params);
+        builder.startObject(Fields.TIERS);
+        for (TierType tierType : TierType.values()) { // fixed order
+            if (tierType != TierType.ON_HEAP) {
+                String tier = tierType.getStringValue();
+                builder.startObject(tier);
+                map.get(tier).toXContent(builder, params);
+                builder.endObject();
+            }
+        }
+        builder.endObject();
         builder.endObject();
         return builder;
     }
@@ -132,6 +173,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
      */
     static final class Fields {
         static final String REQUEST_CACHE_STATS = "request_cache";
+        static final String TIERS = "tiers";
         static final String MEMORY_SIZE = "memory_size";
         static final String MEMORY_SIZE_IN_BYTES = "memory_size_in_bytes";
         static final String EVICTIONS = "evictions";
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index d43418d63a6a6..f8d77a4c0dbaf 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -33,11 +33,9 @@
 package org.opensearch.index.cache.request;
 
 import org.apache.lucene.util.Accountable;
-import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.indices.TierType;
 
-import java.io.Serializable;
 import java.util.EnumMap;
 
 /**
@@ -57,40 +55,11 @@ public ShardRequestCache() {
 
     public RequestCacheStats stats() {
         // TODO: Change RequestCacheStats to support disk tier stats.
-        return stats(TierType.ON_HEAP);
+        // Changing this function to return a RequestCacheStats with stats from all tiers.
+        //return stats(TierType.ON_HEAP);
+        return new RequestCacheStats(statsHolder);
     }
 
-    public RequestCacheStats stats(TierType tierType) {
-        return new RequestCacheStats(
-            statsHolder.get(tierType).totalMetric.count(),
-            statsHolder.get(tierType).evictionsMetric.count(),
-            statsHolder.get(tierType).hitCount.count(),
-            statsHolder.get(tierType).missCount.count(),
-            statsHolder.get(tierType).entries.count()
-        );
-    }
-
-    public RequestCacheStats overallStats() {
-        long totalSize = 0;
-        long totalEvictions = 0;
-        long totalHits = 0;
-        long totalMisses = 0;
-        long totalEntries = 0;
-        for (TierType tierType : TierType.values()) {
-            totalSize += statsHolder.get(tierType).totalMetric.count();
-            totalEvictions += statsHolder.get(tierType).evictionsMetric.count();
-            totalHits += statsHolder.get(tierType).hitCount.count();
-            totalMisses += statsHolder.get(tierType).missCount.count();
-            totalEntries += statsHolder.get(tierType).entries.count();
-        }
-        return new RequestCacheStats(
-            totalSize,
-            totalEvictions,
-            totalHits,
-            totalMisses,
-            totalEntries
-        );
-    }
 
     public void onHit(TierType tierType) {
         statsHolder.get(tierType).hitCount.inc();
@@ -119,13 +88,4 @@ public void onRemoval(Accountable key, BytesReference value, boolean evicted, Ti
         statsHolder.get(tierType).totalMetric.dec(dec);
         statsHolder.get(tierType).entries.dec();
     }
-
-    static class StatsHolder implements Serializable {
-
-        final CounterMetric evictionsMetric = new CounterMetric();
-        final CounterMetric totalMetric = new CounterMetric();
-        final CounterMetric hitCount = new CounterMetric();
-        final CounterMetric missCount = new CounterMetric();
-        final CounterMetric entries = new CounterMetric();
-    }
 }
diff --git a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
new file mode 100644
index 0000000000000..86f99871fea26
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
@@ -0,0 +1,99 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.cache.request;
+
+import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+public class StatsHolder implements Serializable, Writeable, ToXContentFragment {
+    final CounterMetric evictionsMetric;
+    final CounterMetric totalMetric;
+    final CounterMetric hitCount;
+    final CounterMetric missCount;
+    final CounterMetric entries;
+
+
+    public StatsHolder() {
+        this.evictionsMetric = new CounterMetric();
+        this.totalMetric = new CounterMetric();
+        this.hitCount = new CounterMetric();
+        this.missCount = new CounterMetric();
+        this.entries = new CounterMetric();
+    }
+
+    public StatsHolder(StreamInput in) throws IOException  {
+        // Read and write the values of the counter metrics. They should always be positive
+        this.evictionsMetric = new CounterMetric();
+        this.evictionsMetric.inc(in.readVLong());
+        this.totalMetric = new CounterMetric();
+        this.totalMetric.inc(in.readVLong());
+        this.hitCount = new CounterMetric();
+        this.hitCount.inc(in.readVLong());
+        this.missCount = new CounterMetric();
+        this.missCount.inc(in.readVLong());
+        this.entries = new CounterMetric();
+        this.entries.inc(in.readVLong());
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(evictionsMetric.count());
+        out.writeVLong(totalMetric.count());
+        out.writeVLong(hitCount.count());
+        out.writeVLong(missCount.count());
+        out.writeVLong(entries.count());
+    }
+
+    public void add(StatsHolder otherStats) {
+        // Add the argument's metrics to this object's metrics.
+        evictionsMetric.inc(otherStats.evictionsMetric.count());
+        totalMetric.inc(otherStats.totalMetric.count());
+        hitCount.inc(otherStats.hitCount.count());
+        missCount.inc(otherStats.missCount.count());
+        entries.inc(otherStats.entries.count());
+    }
+
+    public long getEvictions() {
+        return evictionsMetric.count();
+    }
+
+    public long getMemorySize() {
+        return totalMetric.count();
+    }
+
+    public long getHitCount() {
+        return hitCount.count();
+    }
+
+    public long getMissCount() {
+        return missCount.count();
+    }
+
+    public long getEntries() {
+        return entries.count();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.humanReadableField(RequestCacheStats.Fields.MEMORY_SIZE_IN_BYTES, RequestCacheStats.Fields.MEMORY_SIZE, new ByteSizeValue(getMemorySize()));
+        builder.field(RequestCacheStats.Fields.EVICTIONS, getEvictions());
+        builder.field(RequestCacheStats.Fields.HIT_COUNT, getHitCount());
+        builder.field(RequestCacheStats.Fields.MISS_COUNT, getMissCount());
+        builder.field(RequestCacheStats.Fields.ENTRIES, getEntries());
+        return builder;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/TierType.java b/server/src/main/java/org/opensearch/indices/TierType.java
index 9a286fd26151b..900df2fce57c5 100644
--- a/server/src/main/java/org/opensearch/indices/TierType.java
+++ b/server/src/main/java/org/opensearch/indices/TierType.java
@@ -10,6 +10,17 @@
 
 public enum TierType {
 
-    ON_HEAP,
-    DISK;
+    ON_HEAP("on_heap"),
+    DISK("disk");
+
+    private final String stringValue;
+
+    TierType(String stringValue) {
+        // Associate each TierType with a string representation, for use in API responses and elsewhere
+        this.stringValue = stringValue;
+    }
+
+    public String getStringValue() {
+        return this.stringValue;
+    }
 }
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 2bd2bd53a787a..6e75a0e8102c2 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -196,8 +196,8 @@ public void testSpillover() throws Exception {
         assertEquals(maxNumInHeap * heapKeySize, requestCacheStats.stats().getMemorySizeInBytes());
         // TODO: disk weight bytes
         assertEquals(1, requestCacheStats.stats().getEvictions());
-        assertEquals(1, requestCacheStats.stats(TierType.DISK).getHitCount());
-        assertEquals(maxNumInHeap + 1, requestCacheStats.stats(TierType.DISK).getMissCount());
+        assertEquals(1, requestCacheStats.stats().getHitCount(TierType.DISK));
+        assertEquals(maxNumInHeap + 1, requestCacheStats.stats().getMissCount(TierType.DISK));
         assertEquals(0, requestCacheStats.stats().getHitCount());
         assertEquals(maxNumInHeap + 2, requestCacheStats.stats().getMissCount());
         assertEquals(maxNumInHeap, cache.tieredCacheHandler.count(TierType.ON_HEAP));
@@ -209,8 +209,8 @@ public void testSpillover() throws Exception {
         BytesReference firstValueAgain = cache.getOrCompute(entity, loader, reader, termBytesArr[0]);
 
         assertEquals(1, requestCacheStats.stats().getEvictions());
-        assertEquals(2, requestCacheStats.stats(TierType.DISK).getHitCount());
-        assertEquals(maxNumInHeap + 1, requestCacheStats.stats(TierType.DISK).getMissCount());
+        assertEquals(2, requestCacheStats.stats().getHitCount(TierType.DISK));
+        assertEquals(maxNumInHeap + 1, requestCacheStats.stats().getMissCount(TierType.DISK));
         assertEquals(1, requestCacheStats.stats().getHitCount());
         assertEquals(maxNumInHeap + 3, requestCacheStats.stats().getMissCount());
         assertEquals(maxNumInHeap, cache.tieredCacheHandler.count(TierType.ON_HEAP));

From 9d8d433197b55dd5a4877485ec6ba4eb4dfcc8e3 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Fri, 20 Oct 2023 16:47:20 -0700
Subject: [PATCH 031/120] Added version checks to streaming functions

---
 .../cache/request/RequestCacheStats.java      | 38 ++++++++++++++-----
 .../index/cache/request/StatsHolder.java      | 21 ++++++----
 .../TieredCacheSpilloverStrategyHandler.java  |  3 +-
 3 files changed, 44 insertions(+), 18 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index 5480dd5b01240..4bdb91ba9e973 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.cache.request;
 
+import org.opensearch.Version;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -61,14 +62,25 @@ public RequestCacheStats() {
     }
 
     public RequestCacheStats(StreamInput in) throws IOException {
-        // Any RequestCacheStats written to a stream should already have a counter for each possible tier type
-        this.map = in.readMap(StreamInput::readString, StatsHolder::new); // does it know to use the right constructor? does it rly need to be registered?
-    }
-
-    public RequestCacheStats(TierType tierType, StatsHolder statsHolder) {
-        // Create a RequestCacheStats object with only one tier's statistics populated
         this();
-        map.put(tierType.getStringValue(), statsHolder);
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            this.map = in.readMap(StreamInput::readString, StatsHolder::new); // does it know to use the right constructor? does it rly need to be registered?
+        } else {
+            // objects from earlier versions only contain on-heap info, and do not have entries info
+            long memorySize = in.readVLong();
+            long evictions = in.readVLong();
+            long hitCount = in.readVLong();
+            long missCount = in.readVLong();
+            this.map.put(
+                TierType.ON_HEAP.getStringValue(),
+                new StatsHolder(
+                    memorySize,
+                    evictions,
+                    hitCount,
+                    missCount,
+                    0
+                ));
+        }
     }
 
     public RequestCacheStats(Map<TierType, StatsHolder> inputMap) {
@@ -91,7 +103,6 @@ private StatsHolder getTierStats(TierType tierType) {
         return map.get(tierType.getStringValue());
     }
 
-    // should these take in strings bc thats whats done in the xcontent builder? seems wasteful
     public long getMemorySizeInBytes(TierType tierType) {
         return getTierStats(tierType).totalMetric.count();
     }
@@ -144,7 +155,16 @@ public long getEntries() {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeMap(this.map, StreamOutput::writeString, (o, v) -> v.writeTo(o)); // ?
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeMap(this.map, StreamOutput::writeString, (o, v) -> v.writeTo(o)); // ?
+        } else {
+            // Write only on-heap values, and don't write entries metric
+            StatsHolder heapStats = map.get(TierType.ON_HEAP.getStringValue());
+            out.writeVLong(heapStats.getMemorySize());
+            out.writeVLong(heapStats.getEvictions());
+            out.writeVLong(heapStats.getHitCount());
+            out.writeVLong(heapStats.getMissCount());
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
index 86f99871fea26..d506d8e78ec7a 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
@@ -35,18 +35,25 @@ public StatsHolder() {
         this.entries = new CounterMetric();
     }
 
-    public StatsHolder(StreamInput in) throws IOException  {
-        // Read and write the values of the counter metrics. They should always be positive
+    public StatsHolder(long evictions, long memorySize, long hitCount, long missCount, long entries) {
         this.evictionsMetric = new CounterMetric();
-        this.evictionsMetric.inc(in.readVLong());
+        this.evictionsMetric.inc(evictions);
         this.totalMetric = new CounterMetric();
-        this.totalMetric.inc(in.readVLong());
+        this.totalMetric.inc(memorySize);
         this.hitCount = new CounterMetric();
-        this.hitCount.inc(in.readVLong());
+        this.hitCount.inc(hitCount);
         this.missCount = new CounterMetric();
-        this.missCount.inc(in.readVLong());
+        this.missCount.inc(missCount);
         this.entries = new CounterMetric();
-        this.entries.inc(in.readVLong());
+        this.entries.inc(entries);
+    }
+
+    public StatsHolder(StreamInput in) throws IOException  {
+        // Read and write the values of the counter metrics. They should always be positive
+        // This object is new, so we shouldn't need version checks for different behavior
+        this(in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong());
+        // java forces us to do this in one line
+        // guaranteed to be evaluated in correct order (https://docs.oracle.com/javase/specs/jls/se7/html/jls-15.html#jls-15.7.4)
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index db9d4ffe41447..b3b4134033c28 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -54,8 +54,6 @@ public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception
             V value = onHeapCachingTier.compute(key, loader);
             tieredCacheEventListener.onCached(key, value, TierType.ON_HEAP);
             return value;
-        } else {
-            //tieredCacheEventListener.onHit(key, cacheValue.value, cacheValue.source); // this double counts, see line 122
         }
         return cacheValue.value;
     }
@@ -105,6 +103,7 @@ public void onRemoval(RemovalNotification<K, V> notification) {
             switch (notification.getTierType()) {
                 case ON_HEAP:
                     diskCachingTier.put(notification.getKey(), notification.getValue());
+
                     break;
                 default:
                     break;

From 7d0356214600dbd6052c5868f5c722c2794fe199 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 23 Oct 2023 12:37:29 -0700
Subject: [PATCH 032/120] Added UT for RequestCacheStats

---
 .../indices/IndicesRequestCacheIT.java        | 43 ++++++---
 .../index/cache/request/StatsHolder.java      | 15 +--
 .../cache/request/RequestCacheStatsTests.java | 95 +++++++++++++++++++
 3 files changed, 134 insertions(+), 19 deletions(-)
 create mode 100644 server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index 36e327ce11b6a..7e48fee7d1e6b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -34,6 +34,7 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.opensearch.action.IndicesRequestIT;
 import org.opensearch.action.admin.indices.alias.Alias;
 import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.opensearch.action.search.SearchResponse;
@@ -43,6 +44,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
@@ -636,16 +638,27 @@ public void testProfileDisableCache() throws Exception {
 
     public void testCacheWithInvalidation() throws Exception {
         Client client = client();
+        //int heapSizeBytes = 2000; // enough to fit 2 queries, as each is 687 B
+
+        Settings.Builder builder = Settings.builder()
+            .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(2000));
+        // Why is it appending "index." to the beginning of the key??
+
+        String heapSizeBytes = builder.get(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey());
+        System.out.println("Current cap = " + heapSizeBytes);
+
+        client.admin().setSettings(Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(2000)));
+
         assertAcked(
             client.admin()
                 .indices()
                 .prepareCreate("index")
                 .setMapping("k", "type=keyword")
                 .setSettings(
-                    Settings.builder()
-                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    builder
                 )
                 .get()
         );
@@ -662,8 +675,9 @@ public void testCacheWithInvalidation() throws Exception {
         resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
         assertSearchResponse(resp);
         // Should expect hit as here as refresh didn't happen
-        assertCacheState(client, "index", 1, 1);
-        assertNumCacheEntries(client, "index", 1);
+        assertCacheState(client, "index", 1, 1, TierType.ON_HEAP);
+        assertCacheState(client, "index", 0, 1, TierType.DISK);
+        assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP);
 
         // Explicit refresh would invalidate cache
         refresh();
@@ -671,12 +685,13 @@ public void testCacheWithInvalidation() throws Exception {
         resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
         assertSearchResponse(resp);
         // Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh)
-        assertCacheState(client, "index", 1, 2);
-        assertNumCacheEntries(client, "index", 1); // Shouldn't it just be the most recent query, since the first one was invalidated? (prob invalidation isnt in yet)
+        assertCacheState(client, "index", 1, 2, TierType.ON_HEAP);
+        assertCacheState(client, "index", 0, 2, TierType.DISK);
+        assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP); // Shouldn't it just be the most recent query, since the first one was invalidated? (prob invalidation isnt in yet)
         // yeah - evictions = 0, its not in yet
     }
 
-    private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
+    private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses, TierType tierType) {
         RequestCacheStats requestCacheStats = client.admin()
             .indices()
             .prepareStats(index)
@@ -686,14 +701,18 @@ private static void assertCacheState(Client client, String index, long expectedH
             .getRequestCache();
         // Check the hit count and miss count together so if they are not
         // correct we can see both values
+        System.out.println("mem size " + requestCacheStats.getMemorySize(tierType));
         assertEquals(
             Arrays.asList(expectedHits, expectedMisses, 0L),
-            Arrays.asList(requestCacheStats.getHitCount(), requestCacheStats.getMissCount(), requestCacheStats.getEvictions())
+            Arrays.asList(requestCacheStats.getHitCount(tierType), requestCacheStats.getMissCount(tierType), requestCacheStats.getEvictions(tierType))
         );
+    }
 
+    private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
+        assertCacheState(client, index, expectedHits, expectedMisses, TierType.ON_HEAP);
     }
 
-    private static void assertNumCacheEntries(Client client, String index, long expectedEntries) {
+    private static void assertNumCacheEntries(Client client, String index, long expectedEntries, TierType tierType) {
         RequestCacheStats requestCacheStats = client.admin()
             .indices()
             .prepareStats(index)
@@ -701,7 +720,7 @@ private static void assertNumCacheEntries(Client client, String index, long expe
             .get()
             .getTotal()
             .getRequestCache();
-        assertEquals(expectedEntries, requestCacheStats.getEntries());
+        assertEquals(expectedEntries, requestCacheStats.getEntries(tierType));
     }
 
 }
diff --git a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
index d506d8e78ec7a..c7d7447a68459 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
@@ -20,26 +20,27 @@
 import java.io.Serializable;
 
 public class StatsHolder implements Serializable, Writeable, ToXContentFragment {
-    final CounterMetric evictionsMetric;
     final CounterMetric totalMetric;
+    final CounterMetric evictionsMetric;
     final CounterMetric hitCount;
     final CounterMetric missCount;
     final CounterMetric entries;
 
 
     public StatsHolder() {
-        this.evictionsMetric = new CounterMetric();
         this.totalMetric = new CounterMetric();
+        this.evictionsMetric = new CounterMetric();
         this.hitCount = new CounterMetric();
         this.missCount = new CounterMetric();
         this.entries = new CounterMetric();
     }
 
-    public StatsHolder(long evictions, long memorySize, long hitCount, long missCount, long entries) {
-        this.evictionsMetric = new CounterMetric();
-        this.evictionsMetric.inc(evictions);
+    public StatsHolder(long memorySize, long evictions, long hitCount, long missCount, long entries) {
+        // Switched argument order to match RequestCacheStats
         this.totalMetric = new CounterMetric();
         this.totalMetric.inc(memorySize);
+        this.evictionsMetric = new CounterMetric();
+        this.evictionsMetric.inc(evictions);
         this.hitCount = new CounterMetric();
         this.hitCount.inc(hitCount);
         this.missCount = new CounterMetric();
@@ -58,8 +59,8 @@ public StatsHolder(StreamInput in) throws IOException  {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVLong(evictionsMetric.count());
         out.writeVLong(totalMetric.count());
+        out.writeVLong(evictionsMetric.count());
         out.writeVLong(hitCount.count());
         out.writeVLong(missCount.count());
         out.writeVLong(entries.count());
@@ -67,8 +68,8 @@ public void writeTo(StreamOutput out) throws IOException {
 
     public void add(StatsHolder otherStats) {
         // Add the argument's metrics to this object's metrics.
-        evictionsMetric.inc(otherStats.evictionsMetric.count());
         totalMetric.inc(otherStats.totalMetric.count());
+        evictionsMetric.inc(otherStats.evictionsMetric.count());
         hitCount.inc(otherStats.hitCount.count());
         missCount.inc(otherStats.missCount.count());
         entries.inc(otherStats.entries.count());
diff --git a/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java b/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
new file mode 100644
index 0000000000000..0b1adbe0f87c7
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
@@ -0,0 +1,95 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.cache.request;
+
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.BytesStream;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.indices.TierType;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class RequestCacheStatsTests extends OpenSearchTestCase {
+    public void testConstructorsAndAdd() throws Exception {
+        RequestCacheStats emptyStats = new RequestCacheStats();
+        for (TierType tierType : TierType.values()) {
+            assertTierState(emptyStats, tierType, 0, 0, 0, 0, 0);
+        }
+        Map<TierType, StatsHolder> testHeapMap = new HashMap<>();
+        testHeapMap.put(TierType.ON_HEAP, new StatsHolder(
+            1, 2, 3, 4, 5
+        ));
+        RequestCacheStats heapOnlyStats = new RequestCacheStats(testHeapMap);
+        for (TierType tierType : TierType.values()) {
+            if (tierType == TierType.ON_HEAP) {
+                assertTierState(heapOnlyStats, tierType, 1, 2, 3, 4, 5);
+            } else {
+                assertTierState(heapOnlyStats, tierType, 0, 0, 0, 0, 0);
+            }
+        }
+
+        Map<TierType, StatsHolder> testBothTiersMap = new HashMap<>();
+        testBothTiersMap.put(TierType.ON_HEAP, new StatsHolder(
+            11, 12, 13, 14, 15
+        ));
+        testBothTiersMap.put(TierType.DISK, new StatsHolder(
+            6, 7, 8, 9, 10
+        ));
+        RequestCacheStats bothTiersStats = new RequestCacheStats(testBothTiersMap);
+        assertTierState(bothTiersStats, TierType.ON_HEAP, 11, 12, 13, 14, 15);
+        assertTierState(bothTiersStats, TierType.DISK, 6, 7, 8, 9, 10);
+
+        bothTiersStats.add(heapOnlyStats);
+        assertTierState(bothTiersStats, TierType.ON_HEAP, 12, 14, 16, 18, 20);
+        assertTierState(bothTiersStats, TierType.DISK, 6, 7, 8, 9, 10);
+    }
+
+    public void testSerialization() throws Exception {
+        // This test also implicitly tests StreamHolder serialization
+        BytesStreamOutput os = new BytesStreamOutput();
+
+        Map<TierType, StatsHolder> testMap = new HashMap<>();
+        testMap.put(TierType.ON_HEAP, new StatsHolder(
+            11, 12, 13, 14, 15
+        ));
+        testMap.put(TierType.DISK, new StatsHolder(
+            6, 7, 8, 9, 10
+        ));
+        RequestCacheStats stats = new RequestCacheStats(testMap);
+        stats.writeTo(os);
+        BytesStreamInput is = new BytesStreamInput(BytesReference.toBytes(os.bytes()));
+        RequestCacheStats deserialized = new RequestCacheStats(is);
+
+        assertTierState(deserialized, TierType.ON_HEAP, 11, 12, 13, 14, 15);
+        assertTierState(deserialized, TierType.DISK, 6, 7, 8, 9, 10);
+    }
+
+    private void assertTierState(
+        RequestCacheStats stats,
+        TierType tierType,
+        long memSize,
+        long evictions,
+        long hitCount,
+        long missCount,
+        long entries
+    ) {
+        assertEquals(memSize, stats.getMemorySizeInBytes(tierType));
+        assertEquals(evictions, stats.getEvictions(tierType));
+        assertEquals(hitCount, stats.getHitCount(tierType));
+        assertEquals(missCount, stats.getMissCount(tierType));
+        assertEquals(entries, stats.getEntries(tierType));
+    }
+
+}

From c5099f1d3c9bd453265b2e0fd05e6344810b1ade Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 23 Oct 2023 15:20:14 -0700
Subject: [PATCH 033/120] Added IT for disk tier stats, plus spotlessApply

---
 .../IndicesRequestCacheDiskTierIT.java        | 88 +++++++++++++++++++
 .../indices/IndicesRequestCacheIT.java        | 72 ++++++++-------
 .../common/metrics/CounterMetric.java         |  1 -
 .../cache/request/RequestCacheStats.java      | 14 +--
 .../cache/request/ShardRequestCache.java      |  3 +-
 .../index/cache/request/StatsHolder.java      |  9 +-
 .../org/opensearch/indices/CachingTier.java   |  2 -
 .../indices/DummySerializableKey.java         |  4 +
 .../indices/EhcacheDiskCachingTier.java       | 74 +++++++++-------
 .../indices/EhcacheEventListener.java         | 12 ++-
 .../indices/IndicesRequestCache.java          |  7 +-
 .../TieredCacheSpilloverStrategyHandler.java  |  1 +
 .../cache/request/RequestCacheStatsTests.java | 27 ++----
 .../indices/IndicesRequestCacheTests.java     |  2 +-
 14 files changed, 204 insertions(+), 112 deletions(-)
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
new file mode 100644
index 0000000000000..47bda50ca788a
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -0,0 +1,88 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse;
+
+// This is a separate file from IndicesRequestCacheIT because we only want to run our test
+// on a node with a maximum request cache size that we set.
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class IndicesRequestCacheDiskTierIT extends OpenSearchIntegTestCase {
+    public void testDiskTierStats() throws Exception {
+        int heapSizeBytes = 1800; // enough to fit 2 queries, as each is 687 B
+        int requestSize = 687; // each request is 687 B
+        String node = internalCluster().startNode(
+            Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+        );
+        Client client = client(node);
+
+        Settings.Builder indicesSettingBuilder = Settings.builder()
+            .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+        assertAcked(
+            client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(indicesSettingBuilder).get()
+        );
+        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+        ensureSearchable("index");
+        SearchResponse resp;
+
+        int numOnDisk = 5;
+        int numRequests = heapSizeBytes / requestSize + numOnDisk;
+        for (int i = 0; i < numRequests; i++) {
+            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+            assertSearchResponse(resp);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
+            System.out.println("request number " + i);
+        }
+
+        System.out.println("Num requests = " + numRequests);
+
+        // the first request, for "hello0", should have been evicted to the disk tier
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello0")).get();
+        IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 1, TierType.ON_HEAP, false);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 1, numRequests, TierType.DISK, false);
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index 7e48fee7d1e6b..2d25079305b17 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -34,7 +34,6 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.opensearch.action.IndicesRequestIT;
 import org.opensearch.action.admin.indices.alias.Alias;
 import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.opensearch.action.search.SearchResponse;
@@ -638,30 +637,13 @@ public void testProfileDisableCache() throws Exception {
 
     public void testCacheWithInvalidation() throws Exception {
         Client client = client();
-        //int heapSizeBytes = 2000; // enough to fit 2 queries, as each is 687 B
 
         Settings.Builder builder = Settings.builder()
             .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(2000));
-        // Why is it appending "index." to the beginning of the key??
-
-        String heapSizeBytes = builder.get(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey());
-        System.out.println("Current cap = " + heapSizeBytes);
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
 
-        client.admin().setSettings(Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(2000)));
-
-        assertAcked(
-            client.admin()
-                .indices()
-                .prepareCreate("index")
-                .setMapping("k", "type=keyword")
-                .setSettings(
-                    builder
-                )
-                .get()
-        );
+        assertAcked(client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(builder).get());
         indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
         ensureSearchable("index");
         SearchResponse resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
@@ -675,8 +657,8 @@ public void testCacheWithInvalidation() throws Exception {
         resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
         assertSearchResponse(resp);
         // Should expect hit as here as refresh didn't happen
-        assertCacheState(client, "index", 1, 1, TierType.ON_HEAP);
-        assertCacheState(client, "index", 0, 1, TierType.DISK);
+        assertCacheState(client, "index", 1, 1, TierType.ON_HEAP, false);
+        assertCacheState(client, "index", 0, 1, TierType.DISK, false);
         assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP);
 
         // Explicit refresh would invalidate cache
@@ -685,13 +667,21 @@ public void testCacheWithInvalidation() throws Exception {
         resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
         assertSearchResponse(resp);
         // Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh)
-        assertCacheState(client, "index", 1, 2, TierType.ON_HEAP);
-        assertCacheState(client, "index", 0, 2, TierType.DISK);
-        assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP); // Shouldn't it just be the most recent query, since the first one was invalidated? (prob invalidation isnt in yet)
+        assertCacheState(client, "index", 1, 2, TierType.ON_HEAP, false);
+        assertCacheState(client, "index", 0, 2, TierType.DISK, false);
+        assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP); // Shouldn't it just be the most recent query, since the first one was
+                                                                     // invalidated? (prob invalidation isnt in yet)
         // yeah - evictions = 0, its not in yet
     }
 
-    private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses, TierType tierType) {
+    protected static void assertCacheState(
+        Client client,
+        String index,
+        long expectedHits,
+        long expectedMisses,
+        TierType tierType,
+        boolean enforceZeroEvictions
+    ) {
         RequestCacheStats requestCacheStats = client.admin()
             .indices()
             .prepareStats(index)
@@ -701,18 +691,32 @@ private static void assertCacheState(Client client, String index, long expectedH
             .getRequestCache();
         // Check the hit count and miss count together so if they are not
         // correct we can see both values
-        System.out.println("mem size " + requestCacheStats.getMemorySize(tierType));
-        assertEquals(
-            Arrays.asList(expectedHits, expectedMisses, 0L),
-            Arrays.asList(requestCacheStats.getHitCount(tierType), requestCacheStats.getMissCount(tierType), requestCacheStats.getEvictions(tierType))
-        );
+        ByteSizeValue memSize = requestCacheStats.getMemorySize(tierType);
+        if (memSize.getBytes() > 0) {
+            System.out.println("mem size " + memSize);
+        }
+        if (enforceZeroEvictions) {
+            assertEquals(
+                Arrays.asList(expectedHits, expectedMisses, 0L),
+                Arrays.asList(
+                    requestCacheStats.getHitCount(tierType),
+                    requestCacheStats.getMissCount(tierType),
+                    requestCacheStats.getEvictions(tierType)
+                )
+            );
+        } else {
+            assertEquals(
+                Arrays.asList(expectedHits, expectedMisses),
+                Arrays.asList(requestCacheStats.getHitCount(tierType), requestCacheStats.getMissCount(tierType))
+            );
+        }
     }
 
-    private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
-        assertCacheState(client, index, expectedHits, expectedMisses, TierType.ON_HEAP);
+    protected static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
+        assertCacheState(client, index, expectedHits, expectedMisses, TierType.ON_HEAP, true);
     }
 
-    private static void assertNumCacheEntries(Client client, String index, long expectedEntries, TierType tierType) {
+    protected static void assertNumCacheEntries(Client client, String index, long expectedEntries, TierType tierType) {
         RequestCacheStats requestCacheStats = client.admin()
             .indices()
             .prepareStats(index)
diff --git a/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java b/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
index 2e5eae5ceebe0..5c48c1f772ff0 100644
--- a/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
+++ b/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
@@ -32,7 +32,6 @@
 
 package org.opensearch.common.metrics;
 
-import java.io.Serializable;
 import java.util.concurrent.atomic.LongAdder;
 
 /**
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index 4bdb91ba9e973..961a000964e98 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -53,6 +53,7 @@
 public class RequestCacheStats implements Writeable, ToXContentFragment {
 
     private Map<String, StatsHolder> map;
+
     public RequestCacheStats() {
         this.map = new HashMap<>();
         for (TierType tierType : TierType.values()) {
@@ -64,22 +65,15 @@ public RequestCacheStats() {
     public RequestCacheStats(StreamInput in) throws IOException {
         this();
         if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
-            this.map = in.readMap(StreamInput::readString, StatsHolder::new); // does it know to use the right constructor? does it rly need to be registered?
+            this.map = in.readMap(StreamInput::readString, StatsHolder::new); // does it know to use the right constructor? does it rly need
+                                                                              // to be registered?
         } else {
             // objects from earlier versions only contain on-heap info, and do not have entries info
             long memorySize = in.readVLong();
             long evictions = in.readVLong();
             long hitCount = in.readVLong();
             long missCount = in.readVLong();
-            this.map.put(
-                TierType.ON_HEAP.getStringValue(),
-                new StatsHolder(
-                    memorySize,
-                    evictions,
-                    hitCount,
-                    missCount,
-                    0
-                ));
+            this.map.put(TierType.ON_HEAP.getStringValue(), new StatsHolder(memorySize, evictions, hitCount, missCount, 0));
         }
     }
 
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index f8d77a4c0dbaf..bfec09cf449da 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -56,11 +56,10 @@ public ShardRequestCache() {
     public RequestCacheStats stats() {
         // TODO: Change RequestCacheStats to support disk tier stats.
         // Changing this function to return a RequestCacheStats with stats from all tiers.
-        //return stats(TierType.ON_HEAP);
+        // return stats(TierType.ON_HEAP);
         return new RequestCacheStats(statsHolder);
     }
 
-
     public void onHit(TierType tierType) {
         statsHolder.get(tierType).hitCount.inc();
     }
diff --git a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
index c7d7447a68459..92d057ab8fd9c 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
@@ -26,7 +26,6 @@ public class StatsHolder implements Serializable, Writeable, ToXContentFragment
     final CounterMetric missCount;
     final CounterMetric entries;
 
-
     public StatsHolder() {
         this.totalMetric = new CounterMetric();
         this.evictionsMetric = new CounterMetric();
@@ -49,7 +48,7 @@ public StatsHolder(long memorySize, long evictions, long hitCount, long missCoun
         this.entries.inc(entries);
     }
 
-    public StatsHolder(StreamInput in) throws IOException  {
+    public StatsHolder(StreamInput in) throws IOException {
         // Read and write the values of the counter metrics. They should always be positive
         // This object is new, so we shouldn't need version checks for different behavior
         this(in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong());
@@ -97,7 +96,11 @@ public long getEntries() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.humanReadableField(RequestCacheStats.Fields.MEMORY_SIZE_IN_BYTES, RequestCacheStats.Fields.MEMORY_SIZE, new ByteSizeValue(getMemorySize()));
+        builder.humanReadableField(
+            RequestCacheStats.Fields.MEMORY_SIZE_IN_BYTES,
+            RequestCacheStats.Fields.MEMORY_SIZE,
+            new ByteSizeValue(getMemorySize())
+        );
         builder.field(RequestCacheStats.Fields.EVICTIONS, getEvictions());
         builder.field(RequestCacheStats.Fields.HIT_COUNT, getHitCount());
         builder.field(RequestCacheStats.Fields.MISS_COUNT, getMissCount());
diff --git a/server/src/main/java/org/opensearch/indices/CachingTier.java b/server/src/main/java/org/opensearch/indices/CachingTier.java
index 6726167fe469d..85596929cfd6b 100644
--- a/server/src/main/java/org/opensearch/indices/CachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/CachingTier.java
@@ -10,8 +10,6 @@
 
 import org.opensearch.common.cache.RemovalListener;
 
-import java.io.IOException;
-
 /**
  * asdsadssa
  * @param <K>
diff --git a/server/src/main/java/org/opensearch/indices/DummySerializableKey.java b/server/src/main/java/org/opensearch/indices/DummySerializableKey.java
index 7f2888f6e65f7..d69734f344d4e 100644
--- a/server/src/main/java/org/opensearch/indices/DummySerializableKey.java
+++ b/server/src/main/java/org/opensearch/indices/DummySerializableKey.java
@@ -14,6 +14,7 @@
 public class DummySerializableKey implements Serializable {
     private Integer i;
     private String s;
+
     public DummySerializableKey(Integer i, String s) {
         this.i = i;
         this.s = s;
@@ -22,9 +23,11 @@ public DummySerializableKey(Integer i, String s) {
     public int getI() {
         return i;
     }
+
     public String getS() {
         return s;
     }
+
     @Override
     public boolean equals(Object o) {
         if (o == this) {
@@ -36,6 +39,7 @@ public boolean equals(Object o) {
         DummySerializableKey other = (DummySerializableKey) o;
         return Objects.equals(this.i, other.i) && this.s.equals(other.s);
     }
+
     @Override
     public final int hashCode() {
         int result = 11;
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
index 2967600ff612a..7d96f587bac65 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
@@ -8,18 +8,8 @@
 
 package org.opensearch.indices;
 
-import org.ehcache.PersistentCacheManager;
-import org.ehcache.config.builders.CacheConfigurationBuilder;
-import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
-import org.ehcache.config.builders.CacheManagerBuilder;
-import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
-import org.ehcache.config.builders.ResourcePoolsBuilder;
-import org.ehcache.config.units.MemoryUnit;
-import org.ehcache.event.EventType;
-import org.ehcache.impl.config.executor.PooledExecutionServiceConfiguration;
 import org.opensearch.common.ExponentiallyWeightedMovingAverage;
 import org.opensearch.common.cache.RemovalListener;
-import org.ehcache.Cache;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.metrics.CounterMetric;
@@ -30,7 +20,21 @@
 import java.util.Collections;
 import java.util.HashMap;
 
-public class EhcacheDiskCachingTier implements DiskCachingTier<IndicesRequestCache.Key, BytesReference>, RemovalListener<IndicesRequestCache.Key, BytesReference> {
+import org.ehcache.Cache;
+import org.ehcache.PersistentCacheManager;
+import org.ehcache.config.builders.CacheConfigurationBuilder;
+import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
+import org.ehcache.config.builders.CacheManagerBuilder;
+import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
+import org.ehcache.config.builders.ResourcePoolsBuilder;
+import org.ehcache.config.units.MemoryUnit;
+import org.ehcache.event.EventType;
+import org.ehcache.impl.config.executor.PooledExecutionServiceConfiguration;
+
+public class EhcacheDiskCachingTier
+    implements
+        DiskCachingTier<IndicesRequestCache.Key, BytesReference>,
+        RemovalListener<IndicesRequestCache.Key, BytesReference> {
 
     public static HashMap<String, PersistentCacheManager> cacheManagers = new HashMap<>();
     // Because of the way test cases are set up, each node may try to instantiate several disk caching tiers.
@@ -52,7 +56,7 @@ public class EhcacheDiskCachingTier implements DiskCachingTier<IndicesRequestCac
     private final String diskCacheFP; // the one to use for this node
     private RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener;
     private ExponentiallyWeightedMovingAverage getTimeMillisEWMA;
-    private static final double GET_TIME_EWMA_ALPHA  = 0.3; // This is the value used elsewhere in OpenSearch
+    private static final double GET_TIME_EWMA_ALPHA = 0.3; // This is the value used elsewhere in OpenSearch
     private static final int MIN_WRITE_THREADS = 0;
     private static final int MAX_WRITE_THREADS = 4; // Max number of threads for the PooledExecutionService which handles writes
     private static final String cacheAlias = "diskTier";
@@ -109,39 +113,42 @@ public void getManager() {
                 // actual logging later
             }
         }
-        PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
+        PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder
+            .newPooledExecutionServiceConfigurationBuilder()
             .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
             .build();
 
-        cacheManagers.put(nodeId,
-            CacheManagerBuilder.newCacheManagerBuilder()
-            .using(threadConfig)
-            .with(CacheManagerBuilder.persistence(diskCacheFP)
-            ).build(true)
+        cacheManagers.put(
+            nodeId,
+            CacheManagerBuilder.newCacheManagerBuilder().using(threadConfig).with(CacheManagerBuilder.persistence(diskCacheFP)).build(true)
         );
         this.cacheManager = cacheManagers.get(nodeId);
     }
 
     private void createCache(long maxWeightInBytes) {
         // our EhcacheEventListener should receive events every time an entry is changed
-        CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder
-            .newEventListenerConfiguration(listener,
-                EventType.EVICTED,
-                EventType.EXPIRED,
-                EventType.REMOVED,
-                EventType.UPDATED,
-                EventType.CREATED)
-            .ordered().asynchronous();
+        CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder.newEventListenerConfiguration(
+            listener,
+            EventType.EVICTED,
+            EventType.EXPIRED,
+            EventType.REMOVED,
+            EventType.UPDATED,
+            EventType.CREATED
+        ).ordered().asynchronous();
         // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
 
-        cache = cacheManager.createCache(cacheAlias,
+        cache = cacheManager.createCache(
+            cacheAlias,
             CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                    EhcacheKey.class, BytesReference.class, ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, false))
-                .withService(listenerConfig));
+                EhcacheKey.class,
+                BytesReference.class,
+                ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, false)
+            ).withService(listenerConfig)
+        );
     }
 
     @Override
-    public BytesReference get(IndicesRequestCache.Key key)  {
+    public BytesReference get(IndicesRequestCache.Key key) {
         // I don't think we need to do the future stuff as the cache is threadsafe
 
         // if (keystore.contains(key.hashCode()) {
@@ -174,7 +181,8 @@ public void put(IndicesRequestCache.Key key, BytesReference value) {
     }
 
     @Override
-    public BytesReference computeIfAbsent(IndicesRequestCache.Key key, TieredCacheLoader<IndicesRequestCache.Key, BytesReference> loader) throws Exception {
+    public BytesReference computeIfAbsent(IndicesRequestCache.Key key, TieredCacheLoader<IndicesRequestCache.Key, BytesReference> loader)
+        throws Exception {
         return null; // should not need to fill out, Cache.computeIfAbsent is always used
     }
 
@@ -193,7 +201,8 @@ public void invalidate(IndicesRequestCache.Key key) {
     }
 
     @Override
-    public BytesReference compute(IndicesRequestCache.Key key, TieredCacheLoader<IndicesRequestCache.Key, BytesReference> loader) throws Exception {
+    public BytesReference compute(IndicesRequestCache.Key key, TieredCacheLoader<IndicesRequestCache.Key, BytesReference> loader)
+        throws Exception {
         return null; // should not need to fill out, Cache.compute is always used
     }
 
@@ -222,6 +231,7 @@ public int count() {
     protected void countInc() {
         count.inc();
     }
+
     protected void countDec() {
         count.dec();
     }
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
index 27872269ba77b..c72fb833a69b6 100644
--- a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
+++ b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
@@ -8,23 +8,26 @@
 
 package org.opensearch.indices;
 
-import org.ehcache.event.CacheEvent;
-import org.ehcache.event.CacheEventListener;
-import org.ehcache.event.EventType;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.core.common.bytes.BytesReference;
 
+import org.ehcache.event.CacheEvent;
+import org.ehcache.event.CacheEventListener;
+import org.ehcache.event.EventType;
+
 public class EhcacheEventListener implements CacheEventListener<EhcacheKey, BytesReference> {
     // Receives key-value pairs (EhcacheKey, BytesReference), but must transform into (Key, BytesReference)
     // to send removal notifications
     private final RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener;
     private final EhcacheDiskCachingTier tier;
+
     EhcacheEventListener(RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener, EhcacheDiskCachingTier tier) {
         this.removalListener = removalListener;
         this.tier = tier; // needed to handle count changes
     }
+
     @Override
     public void onEvent(CacheEvent<? extends EhcacheKey, ? extends BytesReference> event) {
         EhcacheKey ehcacheKey = event.getKey();
@@ -50,7 +53,8 @@ public void onEvent(CacheEvent<? extends EhcacheKey, ? extends BytesReference> e
             case EXPIRED:
             case REMOVED:
                 reason = RemovalReason.INVALIDATED;
-                // this is probably fine for EXPIRED. We use cache.remove() to invalidate keys, but this might overlap with RemovalReason.EXPLICIT?
+                // this is probably fine for EXPIRED. We use cache.remove() to invalidate keys, but this might overlap with
+                // RemovalReason.EXPLICIT?
                 break;
             case UPDATED:
                 reason = RemovalReason.REPLACED;
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index d3084c8c2c231..860875024b18f 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -109,8 +109,10 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
     private final IndicesService indicesService;
     // private final Cache<Key, BytesReference> cache;
 
-    //private final TieredCacheHandler<Key, BytesReference> tieredCacheHandler;
-    public final TieredCacheSpilloverStrategyHandler<Key, BytesReference> tieredCacheHandler; // Change this back after done debugging serialization issues
+    // private final TieredCacheHandler<Key, BytesReference> tieredCacheHandler;
+    public final TieredCacheSpilloverStrategyHandler<Key, BytesReference> tieredCacheHandler; // Change this back after done debugging
+                                                                                              // serialization issues
+
     IndicesRequestCache(Settings settings, IndicesService indicesService) {
         this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
         this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
@@ -140,6 +142,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
     @Override
     public void close() {
         tieredCacheHandler.invalidateAll();
+        tieredCacheHandler.closeDiskTier();
     }
 
     void clear(CacheEntity entity) {
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
index b3b4134033c28..f02a193c7d354 100644
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
@@ -140,6 +140,7 @@ private Function<K, CacheValue<V>> getValueFromTierCache() {
             return null;
         };
     }
+
     @Override
     public void closeDiskTier() {
         diskCachingTier.close();
diff --git a/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java b/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
index 0b1adbe0f87c7..6992b8a441c0a 100644
--- a/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
+++ b/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
@@ -10,14 +10,10 @@
 
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.io.stream.BytesStream;
 import org.opensearch.core.common.io.stream.BytesStreamInput;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.indices.TierType;
 import org.opensearch.test.OpenSearchTestCase;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -28,9 +24,7 @@ public void testConstructorsAndAdd() throws Exception {
             assertTierState(emptyStats, tierType, 0, 0, 0, 0, 0);
         }
         Map<TierType, StatsHolder> testHeapMap = new HashMap<>();
-        testHeapMap.put(TierType.ON_HEAP, new StatsHolder(
-            1, 2, 3, 4, 5
-        ));
+        testHeapMap.put(TierType.ON_HEAP, new StatsHolder(1, 2, 3, 4, 5));
         RequestCacheStats heapOnlyStats = new RequestCacheStats(testHeapMap);
         for (TierType tierType : TierType.values()) {
             if (tierType == TierType.ON_HEAP) {
@@ -41,12 +35,8 @@ public void testConstructorsAndAdd() throws Exception {
         }
 
         Map<TierType, StatsHolder> testBothTiersMap = new HashMap<>();
-        testBothTiersMap.put(TierType.ON_HEAP, new StatsHolder(
-            11, 12, 13, 14, 15
-        ));
-        testBothTiersMap.put(TierType.DISK, new StatsHolder(
-            6, 7, 8, 9, 10
-        ));
+        testBothTiersMap.put(TierType.ON_HEAP, new StatsHolder(11, 12, 13, 14, 15));
+        testBothTiersMap.put(TierType.DISK, new StatsHolder(6, 7, 8, 9, 10));
         RequestCacheStats bothTiersStats = new RequestCacheStats(testBothTiersMap);
         assertTierState(bothTiersStats, TierType.ON_HEAP, 11, 12, 13, 14, 15);
         assertTierState(bothTiersStats, TierType.DISK, 6, 7, 8, 9, 10);
@@ -57,16 +47,12 @@ public void testConstructorsAndAdd() throws Exception {
     }
 
     public void testSerialization() throws Exception {
-        // This test also implicitly tests StreamHolder serialization
+        // This test also implicitly tests StatsHolder serialization
         BytesStreamOutput os = new BytesStreamOutput();
 
         Map<TierType, StatsHolder> testMap = new HashMap<>();
-        testMap.put(TierType.ON_HEAP, new StatsHolder(
-            11, 12, 13, 14, 15
-        ));
-        testMap.put(TierType.DISK, new StatsHolder(
-            6, 7, 8, 9, 10
-        ));
+        testMap.put(TierType.ON_HEAP, new StatsHolder(11, 12, 13, 14, 15));
+        testMap.put(TierType.DISK, new StatsHolder(6, 7, 8, 9, 10));
         RequestCacheStats stats = new RequestCacheStats(testMap);
         stats.writeTo(os);
         BytesStreamInput is = new BytesStreamInput(BytesReference.toBytes(os.bytes()));
@@ -91,5 +77,4 @@ private void assertTierState(
         assertEquals(missCount, stats.getMissCount(tierType));
         assertEquals(entries, stats.getEntries(tierType));
     }
-
 }
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 6e75a0e8102c2..5df6b2154c4d8 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -151,7 +151,7 @@ public void testAddDirectToEhcache() throws Exception {
         String rKey = ((OpenSearchDirectoryReader) reader).getDelegatingCacheHelper().getDelegatingCacheKey().getId().toString();
         IndicesRequestCache.Key key = cache.new Key(entity, termBytes, rKey);
 
-        BytesReference value = new BytesArray(new byte[]{0});
+        BytesReference value = new BytesArray(new byte[] { 0 });
         cache.tieredCacheHandler.getDiskCachingTier().put(key, value);
 
         BytesReference res = cache.tieredCacheHandler.getDiskCachingTier().get(key);

From 64b4bb3acedf967eb9f397777889de96f855ff84 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 23 Oct 2023 15:25:45 -0700
Subject: [PATCH 034/120] cleaned up

---
 .../java/org/opensearch/indices/IndicesRequestCacheIT.java   | 4 ----
 .../opensearch/index/cache/request/RequestCacheStats.java    | 5 +----
 .../opensearch/index/cache/request/ShardRequestCache.java    | 3 ---
 3 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index 2d25079305b17..a9b18c6f8364b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -691,10 +691,6 @@ protected static void assertCacheState(
             .getRequestCache();
         // Check the hit count and miss count together so if they are not
         // correct we can see both values
-        ByteSizeValue memSize = requestCacheStats.getMemorySize(tierType);
-        if (memSize.getBytes() > 0) {
-            System.out.println("mem size " + memSize);
-        }
         if (enforceZeroEvictions) {
             assertEquals(
                 Arrays.asList(expectedHits, expectedMisses, 0L),
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index 961a000964e98..df5e5fee73cb5 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -65,8 +65,7 @@ public RequestCacheStats() {
     public RequestCacheStats(StreamInput in) throws IOException {
         this();
         if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
-            this.map = in.readMap(StreamInput::readString, StatsHolder::new); // does it know to use the right constructor? does it rly need
-                                                                              // to be registered?
+            this.map = in.readMap(StreamInput::readString, StatsHolder::new);
         } else {
             // objects from earlier versions only contain on-heap info, and do not have entries info
             long memorySize = in.readVLong();
@@ -85,8 +84,6 @@ public RequestCacheStats(Map<TierType, StatsHolder> inputMap) {
         }
     }
 
-    // can prob eliminate some of these constructors
-
     public void add(RequestCacheStats stats) {
         for (String tier : stats.map.keySet()) {
             map.get(tier).add(stats.map.get(tier));
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index bfec09cf449da..b72a4d08e1d99 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -54,9 +54,6 @@ public ShardRequestCache() {
     }
 
     public RequestCacheStats stats() {
-        // TODO: Change RequestCacheStats to support disk tier stats.
-        // Changing this function to return a RequestCacheStats with stats from all tiers.
-        // return stats(TierType.ON_HEAP);
         return new RequestCacheStats(statsHolder);
     }
 

From 8d13a171b225422735d4f4325927ad2c41cf9179 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Fri, 22 Sep 2023 13:32:47 -0700
Subject: [PATCH 035/120] Implements a memory-efficient roaring bitmap-based
 keystore for use in disk cache tier

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .gitignore                                    |   3 +-
 server/build.gradle                           |   4 +
 server/licenses/RoaringBitmap-0.9.49.jar.sha1 |   1 +
 server/licenses/RoaringBitmap-LICENSE.txt     | 191 ++++++++++++
 server/licenses/RoaringBitmap-NOTICE.txt      |   0
 server/licenses/shims-0.9.49.jar.sha1         |   1 +
 server/licenses/shims-LICENSE.txt             | 191 ++++++++++++
 server/licenses/shims-NOTICE.txt              |   0
 .../opensearch/indices/KeyLookupStore.java    | 133 ++++++++
 .../indices/RBMIntKeyLookupStore.java         | 264 ++++++++++++++++
 .../opensearch/indices/RBMSizeEstimator.java  | 131 ++++++++
 .../indices/RBMIntKeyLookupStoreTests.java    | 295 ++++++++++++++++++
 12 files changed, 1213 insertions(+), 1 deletion(-)
 create mode 100644 server/licenses/RoaringBitmap-0.9.49.jar.sha1
 create mode 100644 server/licenses/RoaringBitmap-LICENSE.txt
 create mode 100644 server/licenses/RoaringBitmap-NOTICE.txt
 create mode 100644 server/licenses/shims-0.9.49.jar.sha1
 create mode 100644 server/licenses/shims-LICENSE.txt
 create mode 100644 server/licenses/shims-NOTICE.txt
 create mode 100644 server/src/main/java/org/opensearch/indices/KeyLookupStore.java
 create mode 100644 server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java
 create mode 100644 server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java
 create mode 100644 server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java

diff --git a/.gitignore b/.gitignore
index 7514d55cc3c9a..291a63cdeef92 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,4 +64,5 @@ testfixtures_shared/
 .ci/jobs/
 
 # build files generated
-doc-tools/missing-doclet/bin/
\ No newline at end of file
+doc-tools/missing-doclet/bin/
+server/src/main/java/org/opensearch/indices/KLSPerformanceTest.java
diff --git a/server/build.gradle b/server/build.gradle
index f6db3d53a0dcc..426719f53632e 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -158,6 +158,10 @@ dependencies {
   api "com.google.protobuf:protobuf-java:${versions.protobuf}"
   api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}"
 
+  // roaring bitmaps
+  api 'org.roaringbitmap:RoaringBitmap:0.9.49'
+  runtimeOnly 'org.roaringbitmap:shims:0.9.49' // might fix complaining about ArraysShims?
+
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server
     exclude group: 'org.opensearch', module: 'server'
diff --git a/server/licenses/RoaringBitmap-0.9.49.jar.sha1 b/server/licenses/RoaringBitmap-0.9.49.jar.sha1
new file mode 100644
index 0000000000000..919a73c074b6a
--- /dev/null
+++ b/server/licenses/RoaringBitmap-0.9.49.jar.sha1
@@ -0,0 +1 @@
+b45b49c1ec5c5fc48580412d0ca635e1833110ea
\ No newline at end of file
diff --git a/server/licenses/RoaringBitmap-LICENSE.txt b/server/licenses/RoaringBitmap-LICENSE.txt
new file mode 100644
index 0000000000000..a890d4a062fad
--- /dev/null
+++ b/server/licenses/RoaringBitmap-LICENSE.txt
@@ -0,0 +1,191 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and configuration
+files.
+
+"Object" form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object code,
+generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made
+available under the License, as indicated by a copyright notice that is included
+in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version
+of the Work and any modifications or additions to that Work or Derivative Works
+thereof, that is intentionally submitted to Licensor for inclusion in the Work
+by the copyright owner or by an individual or Legal Entity authorized to submit
+on behalf of the copyright owner. For the purposes of this definition,
+"submitted" means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems, and
+issue tracking systems that are managed by, or on behalf of, the Licensor for
+the purpose of discussing and improving the Work, but excluding communication
+that is conspicuously marked or otherwise designated in writing by the copyright
+owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+2. Grant of Copyright License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the Work and such
+Derivative Works in Source or Object form.
+
+3. Grant of Patent License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable (except as stated in this section) patent license to make, have
+made, use, offer to sell, sell, import, and otherwise transfer the Work, where
+such license applies only to those patent claims licensable by such Contributor
+that are necessarily infringed by their Contribution(s) alone or by combination
+of their Contribution(s) with the Work to which such Contribution(s) was
+submitted. If You institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work or a
+Contribution incorporated within the Work constitutes direct or contributory
+patent infringement, then any patent licenses granted to You under this License
+for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution.
+
+You may reproduce and distribute copies of the Work or Derivative Works thereof
+in any medium, with or without modifications, and in Source or Object form,
+provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of
+this License; and
+You must cause any modified files to carry prominent notices stating that You
+changed the files; and
+You must retain, in the Source form of any Derivative Works that You distribute,
+all copyright, patent, trademark, and attribution notices from the Source form
+of the Work, excluding those notices that do not pertain to any part of the
+Derivative Works; and
+If the Work includes a "NOTICE" text file as part of its distribution, then any
+Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents of
+the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License.
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a whole,
+provided Your use, reproduction, and distribution of the Work otherwise complies
+with the conditions stated in this License.
+
+5. Submission of Contributions.
+
+Unless You explicitly state otherwise, any Contribution intentionally submitted
+for inclusion in the Work by You to the Licensor shall be under the terms and
+conditions of this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify the terms of
+any separate license agreement you may have executed with Licensor regarding
+such Contributions.
+
+6. Trademarks.
+
+This License does not grant permission to use the trade names, trademarks,
+service marks, or product names of the Licensor, except as required for
+reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty.
+
+Unless required by applicable law or agreed to in writing, Licensor provides the
+Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
+including, without limitation, any warranties or conditions of TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
+solely responsible for determining the appropriateness of using or
+redistributing the Work and assume any risks associated with Your exercise of
+permissions under this License.
+
+8. Limitation of Liability.
+
+In no event and under no legal theory, whether in tort (including negligence),
+contract, or otherwise, unless required by applicable law (such as deliberate
+and grossly negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special, incidental,
+or consequential damages of any character arising as a result of this License or
+out of the use or inability to use the Work (including but not limited to
+damages for loss of goodwill, work stoppage, computer failure or malfunction, or
+any and all other commercial damages or losses), even if such Contributor has
+been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability.
+
+While redistributing the Work or Derivative Works thereof, You may choose to
+offer, and charge a fee for, acceptance of support, warranty, indemnity, or
+other liability obligations and/or rights consistent with this License. However,
+in accepting such obligations, You may act only on Your own behalf and on Your
+sole responsibility, not on behalf of any other Contributor, and only if You
+agree to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work
+
+To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own
+identifying information. (Don't include the brackets!) The text should be
+enclosed in the appropriate comment syntax for the file format. We also
+recommend that a file or class name and description of purpose be included on
+the same "printed page" as the copyright notice for easier identification within
+third-party archives.
+
+   Copyright 2013-2016 the RoaringBitmap authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/server/licenses/RoaringBitmap-NOTICE.txt b/server/licenses/RoaringBitmap-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/server/licenses/shims-0.9.49.jar.sha1 b/server/licenses/shims-0.9.49.jar.sha1
new file mode 100644
index 0000000000000..9e76614ca5207
--- /dev/null
+++ b/server/licenses/shims-0.9.49.jar.sha1
@@ -0,0 +1 @@
+8bd7794fbdaa9536354dd2d8d961d9503beb9460
\ No newline at end of file
diff --git a/server/licenses/shims-LICENSE.txt b/server/licenses/shims-LICENSE.txt
new file mode 100644
index 0000000000000..a890d4a062fad
--- /dev/null
+++ b/server/licenses/shims-LICENSE.txt
@@ -0,0 +1,191 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and configuration
+files.
+
+"Object" form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object code,
+generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made
+available under the License, as indicated by a copyright notice that is included
+in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version
+of the Work and any modifications or additions to that Work or Derivative Works
+thereof, that is intentionally submitted to Licensor for inclusion in the Work
+by the copyright owner or by an individual or Legal Entity authorized to submit
+on behalf of the copyright owner. For the purposes of this definition,
+"submitted" means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems, and
+issue tracking systems that are managed by, or on behalf of, the Licensor for
+the purpose of discussing and improving the Work, but excluding communication
+that is conspicuously marked or otherwise designated in writing by the copyright
+owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+2. Grant of Copyright License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the Work and such
+Derivative Works in Source or Object form.
+
+3. Grant of Patent License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable (except as stated in this section) patent license to make, have
+made, use, offer to sell, sell, import, and otherwise transfer the Work, where
+such license applies only to those patent claims licensable by such Contributor
+that are necessarily infringed by their Contribution(s) alone or by combination
+of their Contribution(s) with the Work to which such Contribution(s) was
+submitted. If You institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work or a
+Contribution incorporated within the Work constitutes direct or contributory
+patent infringement, then any patent licenses granted to You under this License
+for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution.
+
+You may reproduce and distribute copies of the Work or Derivative Works thereof
+in any medium, with or without modifications, and in Source or Object form,
+provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of
+this License; and
+You must cause any modified files to carry prominent notices stating that You
+changed the files; and
+You must retain, in the Source form of any Derivative Works that You distribute,
+all copyright, patent, trademark, and attribution notices from the Source form
+of the Work, excluding those notices that do not pertain to any part of the
+Derivative Works; and
+If the Work includes a "NOTICE" text file as part of its distribution, then any
+Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents of
+the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License.
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a whole,
+provided Your use, reproduction, and distribution of the Work otherwise complies
+with the conditions stated in this License.
+
+5. Submission of Contributions.
+
+Unless You explicitly state otherwise, any Contribution intentionally submitted
+for inclusion in the Work by You to the Licensor shall be under the terms and
+conditions of this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify the terms of
+any separate license agreement you may have executed with Licensor regarding
+such Contributions.
+
+6. Trademarks.
+
+This License does not grant permission to use the trade names, trademarks,
+service marks, or product names of the Licensor, except as required for
+reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty.
+
+Unless required by applicable law or agreed to in writing, Licensor provides the
+Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
+including, without limitation, any warranties or conditions of TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
+solely responsible for determining the appropriateness of using or
+redistributing the Work and assume any risks associated with Your exercise of
+permissions under this License.
+
+8. Limitation of Liability.
+
+In no event and under no legal theory, whether in tort (including negligence),
+contract, or otherwise, unless required by applicable law (such as deliberate
+and grossly negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special, incidental,
+or consequential damages of any character arising as a result of this License or
+out of the use or inability to use the Work (including but not limited to
+damages for loss of goodwill, work stoppage, computer failure or malfunction, or
+any and all other commercial damages or losses), even if such Contributor has
+been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability.
+
+While redistributing the Work or Derivative Works thereof, You may choose to
+offer, and charge a fee for, acceptance of support, warranty, indemnity, or
+other liability obligations and/or rights consistent with this License. However,
+in accepting such obligations, You may act only on Your own behalf and on Your
+sole responsibility, not on behalf of any other Contributor, and only if You
+agree to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work
+
+To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own
+identifying information. (Don't include the brackets!) The text should be
+enclosed in the appropriate comment syntax for the file format. We also
+recommend that a file or class name and description of purpose be included on
+the same "printed page" as the copyright notice for easier identification within
+third-party archives.
+
+   Copyright 2013-2016 the RoaringBitmap authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/server/licenses/shims-NOTICE.txt b/server/licenses/shims-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/server/src/main/java/org/opensearch/indices/KeyLookupStore.java b/server/src/main/java/org/opensearch/indices/KeyLookupStore.java
new file mode 100644
index 0000000000000..60e1386a460ec
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/KeyLookupStore.java
@@ -0,0 +1,133 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.indices;
+
+/**
+ * An interface for objects that hold an in-memory record of hashes of keys in the disk cache.
+ * These objects have some internal data structure which stores some transformation of added
+ * int values. The internal representations may have collisions. Example transformations include a modulo
+ * or -abs(value), or some combination.
+ */
+public interface KeyLookupStore<T> {
+
+    /**
+     * Transforms the input value into the internal representation for this keystore
+     * and adds it to the internal data structure.
+     * @param value The value to add.
+     * @return true if the value was added, false if it wasn't added because of a
+     * collision or if it was already present.
+     */
+    boolean add(T value) throws Exception;
+
+    /**
+     * Checks if the transformation of the value is in the keystore.
+     * @param value The value to check.
+     * @return true if the value was found, false otherwise. Due to collisions, false positives are
+     * possible, but there should be no false negatives unless forceRemove() is called.
+     */
+    boolean contains(T value) throws Exception;
+
+    /**
+     * Returns the transformed version of the input value, that would be used to stored it in the keystore.
+     * This transformation should be always be the same for a given instance.
+     * @param value The value to transform.
+     * @return The transformed value.
+     */
+    T getInternalRepresentation(T value);
+
+    /**
+     * Attempts to safely remove a value from the internal structure, maintaining the property that contains(value)
+     * will never return a false negative. If removing would lead to a false negative, the value won't be removed.
+     * Classes may not implement safe removal.
+     * @param value The value to attempt to remove.
+     * @return true if the value was removed, false if it wasn't.
+     */
+    boolean remove(T value) throws Exception;
+
+    /**
+     * Returns the number of distinct values stored in the internal data structure.
+     * Does not count values which weren't successfully added due to collisions.
+     * @return The number of values
+     */
+    int getSize();
+
+    /**
+     * Returns the number of times add() has been run, including unsuccessful attempts.
+     * @return The number of adding attempts.
+     */
+    int getTotalAdds();
+
+    /**
+     * Returns the number of times add() has returned false due to a collision.
+     * @return The number of collisions.
+     */
+    int getCollisions();
+
+
+    /**
+     * Checks if two values would collide after being transformed by this store's transformation.
+     * @param value1 The first value to compare.
+     * @param value2 The second value to compare.
+     * @return true if the transformations are equal, false otherwise.
+     */
+    boolean isCollision(T value1, T value2);
+
+    /**
+     * Returns an estimate of the store's memory usage.
+     * @return The memory usage, in MB
+     */
+    long getMemorySizeInBytes();
+
+    /**
+     * Returns the cap for the store's memory usage.
+     * @return The cap, in bytes
+     */
+    long getMemorySizeCapInBytes();
+
+    /**
+     * Returns whether the store is at memory capacity and can't accept more entries
+     */
+    boolean isFull();
+
+    /**
+     * Deletes the internal data structure and regenerates it from the values passed in.
+     * Also resets all stats related to adding.
+     * @param newValues The keys that should be in the reset structure.
+     */
+    void regenerateStore(T[] newValues) throws Exception;
+
+    /**
+     * Deletes all keys and resets all stats related to adding.
+     */
+    void clear() throws Exception;
+}
diff --git a/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java
new file mode 100644
index 0000000000000..3789989b5eaf1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java
@@ -0,0 +1,264 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.metrics.CounterMetric;
+import org.roaringbitmap.RoaringBitmap;
+
+import java.util.HashSet;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This class implements KeyLookupStore<Integer> using a roaring bitmap with a modulo applied to values.
+ * The modulo increases the density of values, which makes RBMs more memory-efficient. The recommended modulo is ~2^28.
+ * It also maintains a hash set of values which have had collisions. Values which haven't had collisions can be
+ * safely removed from the store. The fraction of collided values should be low,
+ * about 0.5% for a store with 10^7 values and a modulo of 2^28.
+ * The store estimates its memory footprint and will stop adding more values once it reaches its memory cap.
+ */
+public class RBMIntKeyLookupStore implements KeyLookupStore<Integer> {
+    protected final int modulo;
+    protected class KeyStoreStats {
+        protected int size;
+        protected long memSizeCapInBytes;
+        protected CounterMetric numAddAttempts;
+        protected CounterMetric numCollisions;
+        protected boolean guaranteesNoFalseNegatives;
+        protected int maxNumEntries;
+        protected boolean atCapacity;
+        protected CounterMetric numRemovalAttempts;
+        protected CounterMetric numSuccessfulRemovals;
+        protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) {
+            this.size = 0;
+            this.numAddAttempts = new CounterMetric();
+            this.numCollisions = new CounterMetric();
+            this.memSizeCapInBytes = memSizeCapInBytes;
+            this.maxNumEntries = maxNumEntries;
+            this.atCapacity = false;
+            this.numRemovalAttempts = new CounterMetric();
+            this.numSuccessfulRemovals = new CounterMetric();
+        }
+    }
+
+    protected KeyStoreStats stats;
+    protected RoaringBitmap rbm;
+    private HashSet<Integer> collidedInts;
+    protected RBMSizeEstimator sizeEstimator;
+    protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    protected final Lock readLock = lock.readLock();
+    protected final Lock writeLock = lock.writeLock();
+
+    RBMIntKeyLookupStore(int modulo, long memSizeCapInBytes) {
+        this.modulo = modulo;
+        sizeEstimator = new RBMSizeEstimator(modulo);
+        this.stats = new KeyStoreStats(memSizeCapInBytes, calculateMaxNumEntries(memSizeCapInBytes));
+        this.rbm = new RoaringBitmap();
+        collidedInts = new HashSet<>();
+    }
+
+    protected int calculateMaxNumEntries(long memSizeCapInBytes) {
+        if (memSizeCapInBytes == 0) {
+            return Integer.MAX_VALUE;
+        }
+        return sizeEstimator.getNumEntriesFromSizeInBytes(memSizeCapInBytes);
+    }
+
+    protected final int transform(int value) {
+        return modulo == 0 ? value : value % modulo;
+    }
+
+    protected void handleCollisions(int transformedValue) {
+        stats.numCollisions.inc();
+        collidedInts.add(transformedValue);
+    }
+
+    @Override
+    public boolean add(Integer value) throws Exception {
+        if (value == null) {
+            return false;
+        }
+        writeLock.lock();
+        stats.numAddAttempts.inc();
+        try {
+            if (stats.size == stats.maxNumEntries) {
+                stats.atCapacity = true;
+                return false;
+            }
+            int transformedValue = transform(value);
+            boolean alreadyContained = contains(value);
+            if (!alreadyContained) {
+                rbm.add(transformedValue);
+                stats.size++;
+                return true;
+            }
+            handleCollisions(transformedValue);
+            return false;
+        } finally {
+            writeLock.unlock();
+        }
+    }
+
+    @Override
+    public boolean contains(Integer value) throws Exception {
+        if (value == null) {
+            return false;
+        }
+        int transformedValue = transform(value);
+        readLock.lock();
+        try {
+            return rbm.contains(transformedValue);
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public Integer getInternalRepresentation(Integer value) {
+        if (value == null) {
+            return 0;
+        }
+        return Integer.valueOf(transform(value));
+    }
+
+    @Override
+    public boolean remove(Integer value) throws Exception {
+        if (value == null) {
+            return false;
+        }
+        int transformedValue = transform(value);
+        readLock.lock();
+        try {
+            if (!contains(value)) {
+                return false;
+            }
+            stats.numRemovalAttempts.inc();
+            if (collidedInts.contains(transformedValue)) {
+                return false;
+            }
+        } finally {
+            readLock.unlock();
+        }
+        writeLock.lock();
+        try {
+            rbm.remove(transformedValue);
+            stats.size--;
+            stats.numSuccessfulRemovals.inc();
+            return true;
+        } finally {
+            writeLock.unlock();
+        }
+    }
+
+    @Override
+    public int getSize() {
+        readLock.lock();
+        try {
+            return stats.size;
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public int getTotalAdds() {
+        return (int) stats.numAddAttempts.count();
+    }
+
+    @Override
+    public int getCollisions() {
+        return (int) stats.numCollisions.count();
+    }
+
+
+    @Override
+    public boolean isCollision(Integer value1, Integer value2) {
+        if (value1 == null || value2 == null) {
+            return false;
+        }
+        return transform(value1) == transform(value2);
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        return sizeEstimator.getSizeInBytes(stats.size) + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size());
+    }
+
+    @Override
+    public long getMemorySizeCapInBytes() {
+        return stats.memSizeCapInBytes;
+    }
+
+    @Override
+    public boolean isFull() {
+        return stats.atCapacity;
+    }
+
+    @Override
+    public void regenerateStore(Integer[] newValues) throws Exception {
+        rbm.clear();
+        collidedInts = new HashSet<>();
+        stats.size = 0;
+        stats.numAddAttempts = new CounterMetric();
+        stats.numCollisions = new CounterMetric();
+        stats.guaranteesNoFalseNegatives = true;
+        stats.numRemovalAttempts = new CounterMetric();
+        stats.numSuccessfulRemovals = new CounterMetric();
+        for (int i = 0; i < newValues.length; i++) {
+            if (newValues[i] != null) {
+                add(newValues[i]);
+            }
+        }
+    }
+
+
+
+    @Override
+    public void clear() throws Exception {
+        regenerateStore(new Integer[]{});
+    }
+    public int getNumRemovalAttempts() {
+        return (int) stats.numRemovalAttempts.count();
+    }
+
+    public int getNumSuccessfulRemovals() {
+        return (int) stats.numSuccessfulRemovals.count();
+    }
+
+    public boolean valueHasHadCollision(Integer value) {
+        if (value == null) {
+            return false;
+        }
+        return collidedInts.contains(transform(value));
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java b/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java
new file mode 100644
index 0000000000000..6e3b8e581ba9f
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java
@@ -0,0 +1,131 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.indices;
+
+/**
+ * A class used to estimate roaring bitmap memory sizes (and hash set sizes).
+ * Values based on experiments with adding randomly distributed integers, which matches the use case for KeyLookupStore.
+ * In this use case, true values are much higher than an RBM's self-reported size, especially for small RBMs: see
+ * https://github.com/RoaringBitmap/RoaringBitmap/issues/257
+ */
+public class RBMSizeEstimator {
+    public static final int BYTES_IN_MB = 1048576;
+    public static final double HASHSET_MEM_SLOPE = 6.46 * Math.pow(10, -5);
+    protected final double slope;
+    protected final double bufferMultiplier;
+    protected final double intercept;
+
+    RBMSizeEstimator(int modulo) {
+        double[] memValues = calculateMemoryCoefficients(modulo);
+        this.bufferMultiplier = memValues[0];
+        this.slope = memValues[1];
+        this.intercept = memValues[2];
+    }
+
+    public static double[] calculateMemoryCoefficients(int modulo) {
+        // Sets up values to help estimate RBM size given a modulo
+        // Returns an array of {bufferMultiplier, slope, intercept}
+
+        double modifiedModulo;
+        if (modulo == 0) {
+            modifiedModulo = 32.0;
+        } else {
+            modifiedModulo = Math.log(modulo) / Math.log(2);
+        }
+        // we "round up" the modulo to the nearest tested value
+        double highCutoff = 29.001; // Floating point makes 29 not work
+        double mediumCutoff = 28.0;
+        double lowCutoff = 26.0;
+        double bufferMultiplier = 1.0;
+        double slope;
+        double intercept;
+        if (modifiedModulo > highCutoff) {
+            // modulo > 2^29
+            bufferMultiplier = 1.2;
+            slope = 0.637;
+            intercept = 3.091;
+        } else if (modifiedModulo > mediumCutoff) {
+            // 2^29 >= modulo > 2^28
+            slope = 0.619;
+            intercept = 2.993;
+        } else if (modifiedModulo > lowCutoff) {
+            // 2^28 >= modulo > 2^26
+            slope = 0.614;
+            intercept = 2.905;
+        } else {
+            slope = 0.628;
+            intercept = 2.603;
+        }
+        return new double[] { bufferMultiplier, slope, intercept };
+    }
+
+    public long getSizeInBytes(int numEntries) {
+        // Based on a linear fit in log-log space, so that we minimize the error as a proportion rather than as
+        // an absolute value. Should be within ~50% of the true value at worst, and should overestimate rather
+        // than underestimate the memory usage
+        return (long) ((long) Math.pow(numEntries, slope) * (long) Math.pow(10, intercept) * bufferMultiplier);
+    }
+
+    public int getNumEntriesFromSizeInBytes(long sizeInBytes) {
+        // This function has some precision issues especially when composed with its inverse:
+        // numEntries = getNumEntriesFromSizeInBytes(getSizeInBytes(numEntries))
+        // In this case the result can be off by up to a couple percent
+        // However, this shouldn't really matter as both functions are based on memory estimates with higher errors than a couple percent
+        // and this composition won't happen outside of tests
+        return (int) Math.pow(sizeInBytes / (bufferMultiplier * Math.pow(10, intercept)), 1 / slope);
+
+    }
+
+    public static long getSizeInBytesWithModulo(int numEntries, int modulo) {
+        double[] memValues = calculateMemoryCoefficients(modulo);
+        return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]);
+    }
+
+    public static int getNumEntriesFromSizeInBytesWithModulo(long sizeInBytes, int modulo) {
+        double[] memValues = calculateMemoryCoefficients(modulo);
+        return (int) Math.pow(sizeInBytes / (memValues[0] * Math.pow(10, memValues[2])), 1 / memValues[1]);
+    }
+
+
+    protected static long convertMBToBytes(double valMB) {
+        return (long) (valMB * BYTES_IN_MB);
+    }
+
+    protected static double convertBytesToMB(long valBytes) {
+        return (double) valBytes / BYTES_IN_MB;
+    }
+
+    protected static long getHashsetMemSizeInBytes(int numEntries) {
+        return convertMBToBytes(HASHSET_MEM_SLOPE * numEntries);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java
new file mode 100644
index 0000000000000..c857c1ecab768
--- /dev/null
+++ b/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java
@@ -0,0 +1,295 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.Randomness;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.ArrayList;
+import java.util.Random;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+
+public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase {
+    public void testInit() {
+        long memCap = 100 * RBMSizeEstimator.BYTES_IN_MB;
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), memCap);
+        assertEquals(0, kls.getSize());
+        assertEquals(memCap, kls.getMemorySizeCapInBytes());
+    }
+    public void testTransformationLogic() throws Exception {
+        int modulo = (int) Math.pow(2, 29);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+        int offset = 3;
+        for (int i = 0; i < 4; i++) { // after this we run into max value, but thats not a flaw with the class design
+            int posValue = i * modulo + offset;
+            kls.add(posValue);
+            int negValue = -(i * modulo + offset);
+            kls.add(negValue);
+        }
+        assertEquals(2, kls.getSize());
+        int[] testVals = new int[]{0, 1, -1, -23495, 23058, modulo, -modulo, Integer.MAX_VALUE, Integer.MIN_VALUE};
+        for (int value : testVals) {
+            assertTrue(kls.getInternalRepresentation(value) < modulo);
+            assertTrue(kls.getInternalRepresentation(value) > -modulo);
+        }
+    }
+
+    public void testContains() throws Exception {
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+        for (int i = 0; i < 2000; i++) {
+            kls.add(i);
+            assertTrue(kls.contains(i));
+        }
+    }
+
+    public void testAddingStatsGetters() throws Exception {
+        int modulo = (int) Math.pow(2, 15);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, 0L);
+        kls.add(15);
+        kls.add(-15);
+        assertEquals(2, kls.getTotalAdds());
+        assertEquals(0, kls.getCollisions());
+
+        int offset = 1;
+        for (int i = 0; i < 10; i++) {
+            kls.add(i * modulo + offset);
+        }
+        assertEquals(12, kls.getTotalAdds());
+        assertEquals(9, kls.getCollisions());
+    }
+
+    public void testRegenerateStore() throws Exception {
+        int numToAdd = 10000000;
+        Random rand = Randomness.get();
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+        for (int i = 0; i < numToAdd; i++) {
+            kls.add(i);
+        }
+        assertEquals(numToAdd, kls.getSize());
+        Integer[] newVals = new Integer[1000]; // margin accounts for collisions
+        for (int j = 0; j < newVals.length; j++) {
+            newVals[j] = rand.nextInt();
+        }
+        kls.regenerateStore(newVals);
+        assertTrue(Math.abs(kls.getSize() - newVals.length) < 3); // inexact due to collisions
+
+        // test clear()
+        kls.clear();
+        assertEquals(0, kls.getSize());
+    }
+
+    public void testAddingDuplicates() throws Exception {
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+        int numToAdd = 4820411;
+        for (int i = 0; i < numToAdd; i++) {
+            kls.add(i);
+            kls.add(i);
+        }
+        for (int j = 0; j < 1000; j++) {
+            kls.add(577);
+        }
+        assertEquals(numToAdd, kls.getSize());
+    }
+
+    public void testMemoryCapBlocksAdd() throws Exception {
+        int modulo = (int) Math.pow(2, 29);
+        for (int maxEntries: new int[]{2342000, 1000, 100000}) {
+            long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModulo(maxEntries, modulo);
+            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memSizeCapInBytes);
+            for (int j = 0; j < maxEntries + 1000; j++) {
+                kls.add(j);
+            }
+            assertTrue(Math.abs(maxEntries - kls.getSize()) < (double) maxEntries / 25);
+            // exact cap varies a small amount bc of floating point, especially when we use bytes instead of MB for calculations
+            // precision gets much worse when we compose the two functions, as we do here, but this wouldn't happen in an actual use case
+        }
+    }
+
+    public void testConcurrency() throws Exception {
+        Random rand = Randomness.get();
+        for (int j = 0; j < 5; j++) { // test with different numbers of threads
+            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+            int numThreads = rand.nextInt(50) + 1;
+            ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(numThreads);
+            // In this test we want to add the first 200K numbers and check they're all correctly there.
+            // We do some duplicates too to ensure those aren't incorrectly added.
+            int amountToAdd = 200000;
+            ArrayList<Future<Boolean>> wasAdded = new ArrayList<>(amountToAdd);
+            ArrayList<Future<Boolean>> duplicatesWasAdded = new ArrayList<>();
+            for (int i = 0; i < amountToAdd; i++) {
+                wasAdded.add(null);
+            }
+            for (int i = 0; i < amountToAdd; i++) {
+                final int val = i;
+                Future<Boolean> fut = executor.submit(() -> {
+                    boolean didAdd;
+                    try {
+                        didAdd = kls.add(val);
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                    return didAdd;
+                });
+                wasAdded.set(val, fut);
+                if (val % 1000 == 0) {
+                    // do a duplicate add
+                    Future<Boolean> duplicateFut = executor.submit(() -> {
+                        boolean didAdd;
+                        try {
+                            didAdd = kls.add(val);
+                        } catch (Exception e) {
+                            throw new RuntimeException(e);
+                        }
+                        return didAdd;
+                    });
+                    duplicatesWasAdded.add(duplicateFut);
+                }
+            }
+            int originalAdds = 0;
+            int duplicateAdds = 0;
+            for (Future<Boolean> fut : wasAdded) {
+                if (fut.get()) {
+                    originalAdds++;
+                }
+            }
+            for (Future<Boolean> duplicateFut : duplicatesWasAdded) {
+                if (duplicateFut.get()) {
+                    duplicateAdds++;
+                }
+            }
+            for (int i = 0; i < amountToAdd; i++) {
+                assertTrue(kls.contains(i));
+            }
+            assertEquals(amountToAdd, originalAdds + duplicateAdds);
+            assertEquals(amountToAdd, kls.getSize());
+            assertEquals(amountToAdd / 1000, kls.getCollisions());
+            executor.shutdown();
+        }
+    }
+
+    public void testRemoveNoCollisions() throws Exception {
+        long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB;
+        int numToAdd = 195000;
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0, memCap);
+        // there should be no collisions for sequential positive numbers up to modulo
+        for (int i = 0; i < numToAdd; i++) {
+            kls.add(i);
+        }
+        for (int i = 0; i < 1000; i++) {
+            assertTrue(kls.remove(i));
+            assertFalse(kls.contains(i));
+            assertFalse(kls.valueHasHadCollision(i));
+        }
+        assertEquals(numToAdd - 1000, kls.getSize());
+    }
+
+    public void testRemoveWithCollisions() throws Exception {
+        int modulo = (int) Math.pow(2, 26);
+        long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB;
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memCap);
+        for (int i = 0; i < 10; i++) {
+            kls.add(i);
+            if (i % 2 == 1) {
+                kls.add(-i);
+                assertFalse(kls.valueHasHadCollision(i));
+                kls.add(i + modulo);
+                assertTrue(kls.valueHasHadCollision(i));
+            } else {
+                assertFalse(kls.valueHasHadCollision(i));
+            }
+        }
+        assertEquals(15, kls.getSize());
+        for (int i = 0; i < 10; i++) {
+            boolean didRemove = kls.remove(i);
+            if (i % 2 == 1) {
+                // we expect a collision with i + modulo, so we can't remove
+                assertFalse(didRemove);
+                assertTrue(kls.contains(i));
+                // but we should be able to remove -i
+                boolean didRemoveNegative = kls.remove(-i);
+                assertTrue(didRemoveNegative);
+                assertFalse(kls.contains(-i));
+            } else {
+                // we expect no collision
+                assertTrue(didRemove);
+                assertFalse(kls.contains(i));
+                assertFalse(kls.valueHasHadCollision(i));
+            }
+        }
+        assertEquals(5, kls.getSize());
+        int offset = 12;
+        kls.add(offset);
+        for (int j = 1; j < 5; j++) {
+            kls.add(offset + j * modulo);
+        }
+        assertEquals(6, kls.getSize());
+        assertFalse(kls.remove(offset + modulo));
+        assertTrue(kls.valueHasHadCollision(offset + 15 * modulo));
+        assertTrue(kls.contains(offset + 17 * modulo));
+    }
+
+    public void testNullInputs() throws Exception {
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+        assertFalse(kls.add(null));
+        assertFalse(kls.contains(null));
+        assertEquals(0, (int) kls.getInternalRepresentation(null));
+        assertFalse(kls.remove(null));
+        assertFalse(kls.isCollision(null, null));
+        assertEquals(0, kls.getTotalAdds());
+        Integer[] newVals = new Integer[]{1, 17, -2, null, -4, null};
+        kls.regenerateStore(newVals);
+        assertEquals(4, kls.getSize());
+    }
+
+    public void testMemoryCapValueInitialization() {
+        double[] logModulos = new double[] { 0.0, 31.2, 30, 29, 28, 13 };
+        double[] expectedMultipliers = new double[] { 1.2, 1.2, 1.2, 1, 1, 1 };
+        double[] expectedSlopes = new double[] { 0.637, 0.637, 0.637, 0.619, 0.614, 0.629 };
+        double[] expectedIntercepts = new double[] { 3.091, 3.091, 3.091, 2.993, 2.905, 2.603 };
+        long memSizeCapInBytes = (long) 100.0 * RBMSizeEstimator.BYTES_IN_MB;
+        double delta = 0.01;
+        for (int i = 0; i < logModulos.length; i++) {
+            int modulo = 0;
+            if (logModulos[i] != 0) {
+                modulo = (int) Math.pow(2, logModulos[i]);
+            }
+            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memSizeCapInBytes);
+            assertEquals(kls.stats.memSizeCapInBytes, kls.getMemorySizeCapInBytes(), 1.0);
+            assertEquals(expectedMultipliers[i], kls.sizeEstimator.bufferMultiplier, delta);
+            assertEquals(expectedSlopes[i], kls.sizeEstimator.slope, delta);
+            assertEquals(expectedIntercepts[i], kls.sizeEstimator.intercept, delta);
+        }
+
+    }
+}

From 08a4f6d801c9043aa370017a67810d9f0b140204 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 9 Oct 2023 09:59:41 -0700
Subject: [PATCH 036/120] Adds and tests took time for QuerySearchResult

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../opensearch/search/query/QueryPhase.java   |   4 +-
 .../search/query/QuerySearchResult.java       |  11 ++
 .../search/query/QueryPhaseTests.java         | 116 ++++++++++++++++++
 .../search/query/QuerySearchResultTests.java  |   2 +
 4 files changed, 132 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhase.java b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
index f3cf2c13ecdef..14b239d3270d4 100644
--- a/server/src/main/java/org/opensearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
@@ -131,6 +131,7 @@ public void preProcess(SearchContext context) {
     }
 
     public void execute(SearchContext searchContext) throws QueryPhaseExecutionException {
+        long startTime = System.nanoTime();
         if (searchContext.hasOnlySuggest()) {
             suggestProcessor.process(searchContext);
             searchContext.queryResult()
@@ -138,6 +139,7 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep
                     new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Lucene.EMPTY_SCORE_DOCS), Float.NaN),
                     new DocValueFormat[0]
                 );
+            searchContext.queryResult().setTookTimeNanos(System.nanoTime() - startTime);
             return;
         }
 
@@ -165,6 +167,7 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep
             );
             searchContext.queryResult().profileResults(shardResults);
         }
+        searchContext.queryResult().setTookTimeNanos(System.nanoTime() - startTime);
     }
 
     // making public for testing
@@ -292,7 +295,6 @@ static boolean executeInternal(SearchContext searchContext, QueryPhaseSearcher q
                     queryResult.nodeQueueSize(rExecutor.getCurrentQueueSize());
                     queryResult.serviceTimeEWMA((long) rExecutor.getTaskExecutionEWMA());
                 }
-
                 return shouldRescore;
             } finally {
                 // Search phase has finished, no longer need to check for timeout
diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
index 7de605a244d09..5a8cf06bd8b0a 100644
--- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
@@ -87,6 +87,7 @@ public final class QuerySearchResult extends SearchPhaseResult {
     private int nodeQueueSize = -1;
 
     private final boolean isNull;
+    private long tookTimeNanos;
 
     public QuerySearchResult() {
         this(false);
@@ -364,6 +365,7 @@ public void readFromWithId(ShardSearchContextId id, StreamInput in) throws IOExc
         nodeQueueSize = in.readInt();
         setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new));
         setRescoreDocIds(new RescoreDocIds(in));
+        tookTimeNanos = in.readVLong();
     }
 
     @Override
@@ -406,6 +408,7 @@ public void writeToNoId(StreamOutput out) throws IOException {
         out.writeInt(nodeQueueSize);
         out.writeOptionalWriteable(getShardSearchRequest());
         getRescoreDocIds().writeTo(out);
+        out.writeVLong(tookTimeNanos); // VLong as took time should always be positive
     }
 
     public TotalHits getTotalHits() {
@@ -415,4 +418,12 @@ public TotalHits getTotalHits() {
     public float getMaxScore() {
         return maxScore;
     }
+
+    public long getTookTimeNanos() {
+        return tookTimeNanos;
+    }
+
+    public void setTookTimeNanos(long tookTime) {
+        tookTimeNanos = tookTime;
+    }
 }
diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
index 39126a607f968..93c54f93801ab 100644
--- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
@@ -85,9 +85,14 @@
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
+import org.opensearch.action.OriginalIndices;
+import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchShardTask;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.Strings;
+import org.opensearch.core.index.Index;
+import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.tasks.TaskCancelledException;
 import org.opensearch.index.mapper.DateFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
@@ -103,9 +108,11 @@
 import org.opensearch.lucene.queries.MinDocQuery;
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.collapse.CollapseBuilder;
+import org.opensearch.search.internal.AliasFilter;
 import org.opensearch.search.internal.ContextIndexSearcher;
 import org.opensearch.search.internal.ScrollContext;
 import org.opensearch.search.internal.SearchContext;
+import org.opensearch.search.internal.ShardSearchRequest;
 import org.opensearch.search.sort.SortAndFormats;
 import org.opensearch.test.TestSearchContext;
 import org.opensearch.threadpool.ThreadPool;
@@ -115,6 +122,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -1145,6 +1153,114 @@ public void testQueryTimeoutChecker() throws Exception {
         createTimeoutCheckerThenWaitThenRun(timeCacheLifespan / 4, timeCacheLifespan / 2 + timeTolerance, false, true);
     }
 
+    public void testQuerySearchResultTookTime() throws IOException {
+        int sleepMillis = randomIntBetween(500, 4000); // between 0.5 and 4 sec
+        DelayedQueryPhaseSearcher delayedQueryPhaseSearcher = new DelayedQueryPhaseSearcher(sleepMillis);
+
+        // we need to test queryPhase.execute(), not executeInternal(), since that's what the timer wraps around
+        // for that we must set up a searchContext with more functionality than the TestSearchContext,
+        // which requires a bit of complexity with test classes
+
+        Directory dir = newDirectory();
+        final Sort sort = new Sort(new SortField("rank", SortField.Type.INT));
+        IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort);
+        RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+        Document doc = new Document();
+        for (int i = 0; i < 10; i++) {
+            doc.add(new StringField("foo", Integer.toString(i), Store.NO));
+        }
+        w.addDocument(doc);
+        w.close();
+        IndexReader reader = DirectoryReader.open(dir);
+
+        QueryShardContext queryShardContext = mock(QueryShardContext.class);
+        when(queryShardContext.fieldMapper("user")).thenReturn(
+            new NumberFieldType("user", NumberType.INTEGER, true, false, true, false, null, Collections.emptyMap())
+        );
+
+        Index index = new Index("IndexName", "UUID");
+        ShardId shardId = new ShardId(index, 0);
+        long nowInMillis = System.currentTimeMillis();
+        String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(3, 10);
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.allowPartialSearchResults(randomBoolean());
+        ShardSearchRequest request = new ShardSearchRequest(
+            OriginalIndices.NONE,
+            searchRequest,
+            shardId,
+            1,
+            AliasFilter.EMPTY,
+            1f,
+            nowInMillis,
+            clusterAlias,
+            Strings.EMPTY_ARRAY
+        );
+        TestSearchContextWithRequest searchContext = new TestSearchContextWithRequest(
+            queryShardContext,
+            indexShard,
+            newEarlyTerminationContextSearcher(reader, 0, executor),
+            request
+        );
+
+        QueryPhase queryPhase = new QueryPhase(delayedQueryPhaseSearcher);
+        queryPhase.execute(searchContext);
+        long tookTime = searchContext.queryResult().getTookTimeNanos();
+        assertTrue(tookTime >= (long) sleepMillis * 1000000);
+        reader.close();
+        dir.close();
+    }
+
+    private class TestSearchContextWithRequest extends TestSearchContext {
+        ShardSearchRequest request;
+        Query query;
+
+        public TestSearchContextWithRequest(
+            QueryShardContext queryShardContext,
+            IndexShard indexShard,
+            ContextIndexSearcher searcher,
+            ShardSearchRequest request
+        ) {
+            super(queryShardContext, indexShard, searcher);
+            this.request = request;
+            this.query = new TermQuery(new Term("foo", "bar"));
+        }
+
+        @Override
+        public ShardSearchRequest request() {
+            return request;
+        }
+
+        @Override
+        public Query query() {
+            return this.query;
+        }
+    }
+
+    private class DelayedQueryPhaseSearcher extends QueryPhase.DefaultQueryPhaseSearcher implements QueryPhaseSearcher {
+        // add delay into searchWith
+        private final int sleepMillis;
+
+        public DelayedQueryPhaseSearcher(int sleepMillis) {
+            super();
+            this.sleepMillis = sleepMillis;
+        }
+
+        @Override
+        public boolean searchWith(
+            SearchContext searchContext,
+            ContextIndexSearcher searcher,
+            Query query,
+            LinkedList<QueryCollectorContext> collectors,
+            boolean hasFilterCollector,
+            boolean hasTimeout
+        ) throws IOException {
+            try {
+                Thread.sleep(sleepMillis);
+            } catch (Exception ignored) {}
+            return super.searchWith(searchContext, searcher, query, collectors, hasFilterCollector, hasTimeout);
+        }
+    }
+
     private void createTimeoutCheckerThenWaitThenRun(
         long timeout,
         long sleepAfterCreation,
diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
index 41e4e1ae45a73..0d83777dbd4cf 100644
--- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
@@ -99,6 +99,7 @@ private static QuerySearchResult createTestInstance() throws Exception {
         if (randomBoolean()) {
             result.aggregations(InternalAggregationsTests.createTestInstance());
         }
+        assertEquals(0, result.getTookTimeNanos());
         return result;
     }
 
@@ -118,6 +119,7 @@ public void testSerialization() throws Exception {
             assertEquals(aggs.asList(), deserializedAggs.asList());
         }
         assertEquals(querySearchResult.terminatedEarly(), deserialized.terminatedEarly());
+        assertEquals(querySearchResult.getTookTimeNanos(), deserialized.getTookTimeNanos());
     }
 
     public void testNullResponse() throws Exception {

From dfd9128f331bafb58a264e3c4f5bfc6c8fa2f0c5 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 18 Oct 2023 17:48:44 -0700
Subject: [PATCH 037/120] Addressed Ankit's comments

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../org/opensearch/search/query/QueryPhase.java    |  2 +-
 .../opensearch/search/query/QuerySearchResult.java | 14 +++++++++++---
 .../opensearch/search/query/QueryPhaseTests.java   |  2 +-
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhase.java b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
index 14b239d3270d4..e6c281fdf74f4 100644
--- a/server/src/main/java/org/opensearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
@@ -131,7 +131,7 @@ public void preProcess(SearchContext context) {
     }
 
     public void execute(SearchContext searchContext) throws QueryPhaseExecutionException {
-        long startTime = System.nanoTime();
+        final long startTime = System.nanoTime();
         if (searchContext.hasOnlySuggest()) {
             suggestProcessor.process(searchContext);
             searchContext.queryResult()
diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
index 5a8cf06bd8b0a..bb484214325dd 100644
--- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
@@ -34,6 +34,7 @@
 
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.TotalHits;
+import org.opensearch.Version;
 import org.opensearch.common.io.stream.DelayableWriteable;
 import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -88,6 +89,7 @@ public final class QuerySearchResult extends SearchPhaseResult {
 
     private final boolean isNull;
     private long tookTimeNanos;
+    private final static Version minimumTookTimeVersion = Version.V_3_0_0;
 
     public QuerySearchResult() {
         this(false);
@@ -365,7 +367,11 @@ public void readFromWithId(ShardSearchContextId id, StreamInput in) throws IOExc
         nodeQueueSize = in.readInt();
         setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new));
         setRescoreDocIds(new RescoreDocIds(in));
-        tookTimeNanos = in.readVLong();
+        if (in.getVersion().onOrAfter(minimumTookTimeVersion)) {
+            tookTimeNanos = in.readVLong();
+        } else {
+            tookTimeNanos = -1L;
+        }
     }
 
     @Override
@@ -408,7 +414,9 @@ public void writeToNoId(StreamOutput out) throws IOException {
         out.writeInt(nodeQueueSize);
         out.writeOptionalWriteable(getShardSearchRequest());
         getRescoreDocIds().writeTo(out);
-        out.writeVLong(tookTimeNanos); // VLong as took time should always be positive
+        if (out.getVersion().onOrAfter(minimumTookTimeVersion)) {
+            out.writeVLong(tookTimeNanos); // VLong as took time should always be positive
+        }
     }
 
     public TotalHits getTotalHits() {
@@ -423,7 +431,7 @@ public long getTookTimeNanos() {
         return tookTimeNanos;
     }
 
-    public void setTookTimeNanos(long tookTime) {
+    void setTookTimeNanos(long tookTime) {
         tookTimeNanos = tookTime;
     }
 }
diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
index 93c54f93801ab..7aa37a4cbe342 100644
--- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
@@ -1154,7 +1154,7 @@ public void testQueryTimeoutChecker() throws Exception {
     }
 
     public void testQuerySearchResultTookTime() throws IOException {
-        int sleepMillis = randomIntBetween(500, 4000); // between 0.5 and 4 sec
+        int sleepMillis = randomIntBetween(10, 100); // between 0.01 and 0.1 sec
         DelayedQueryPhaseSearcher delayedQueryPhaseSearcher = new DelayedQueryPhaseSearcher(sleepMillis);
 
         // we need to test queryPhase.execute(), not executeInternal(), since that's what the timer wraps around

From a9ab3278c9c8cb748c1fc0ab4a285832a0faab98 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 19 Oct 2023 21:36:39 -0700
Subject: [PATCH 038/120] Removed minimum version variable

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../java/org/opensearch/search/query/QuerySearchResult.java  | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
index bb484214325dd..d3bfbea847169 100644
--- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
@@ -89,7 +89,6 @@ public final class QuerySearchResult extends SearchPhaseResult {
 
     private final boolean isNull;
     private long tookTimeNanos;
-    private final static Version minimumTookTimeVersion = Version.V_3_0_0;
 
     public QuerySearchResult() {
         this(false);
@@ -367,7 +366,7 @@ public void readFromWithId(ShardSearchContextId id, StreamInput in) throws IOExc
         nodeQueueSize = in.readInt();
         setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new));
         setRescoreDocIds(new RescoreDocIds(in));
-        if (in.getVersion().onOrAfter(minimumTookTimeVersion)) {
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
             tookTimeNanos = in.readVLong();
         } else {
             tookTimeNanos = -1L;
@@ -414,7 +413,7 @@ public void writeToNoId(StreamOutput out) throws IOException {
         out.writeInt(nodeQueueSize);
         out.writeOptionalWriteable(getShardSearchRequest());
         getRescoreDocIds().writeTo(out);
-        if (out.getVersion().onOrAfter(minimumTookTimeVersion)) {
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
             out.writeVLong(tookTimeNanos); // VLong as took time should always be positive
         }
     }

From 4e57f4c03779160d54842a2db628e174ed583724 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 26 Oct 2023 12:07:27 -0700
Subject: [PATCH 039/120] Changed type for tookTime from long to Long, to
 address Sagar's comment

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../search/query/QuerySearchResult.java       | 10 ++---
 .../search/query/QueryPhaseTests.java         |  2 +-
 .../search/query/QuerySearchResultTests.java  | 40 +++++++++++--------
 3 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
index d3bfbea847169..2ad601f3bf4a6 100644
--- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
@@ -88,7 +88,7 @@ public final class QuerySearchResult extends SearchPhaseResult {
     private int nodeQueueSize = -1;
 
     private final boolean isNull;
-    private long tookTimeNanos;
+    private Long tookTimeNanos = null;
 
     public QuerySearchResult() {
         this(false);
@@ -367,9 +367,9 @@ public void readFromWithId(ShardSearchContextId id, StreamInput in) throws IOExc
         setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new));
         setRescoreDocIds(new RescoreDocIds(in));
         if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
-            tookTimeNanos = in.readVLong();
+            tookTimeNanos = in.readOptionalLong();
         } else {
-            tookTimeNanos = -1L;
+            tookTimeNanos = null;
         }
     }
 
@@ -414,7 +414,7 @@ public void writeToNoId(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(getShardSearchRequest());
         getRescoreDocIds().writeTo(out);
         if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
-            out.writeVLong(tookTimeNanos); // VLong as took time should always be positive
+            out.writeOptionalLong(tookTimeNanos);
         }
     }
 
@@ -426,7 +426,7 @@ public float getMaxScore() {
         return maxScore;
     }
 
-    public long getTookTimeNanos() {
+    public Long getTookTimeNanos() {
         return tookTimeNanos;
     }
 
diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
index 7aa37a4cbe342..ef30cea39be5c 100644
--- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
@@ -1204,7 +1204,7 @@ public void testQuerySearchResultTookTime() throws IOException {
 
         QueryPhase queryPhase = new QueryPhase(delayedQueryPhaseSearcher);
         queryPhase.execute(searchContext);
-        long tookTime = searchContext.queryResult().getTookTimeNanos();
+        Long tookTime = searchContext.queryResult().getTookTimeNanos();
         assertTrue(tookTime >= (long) sleepMillis * 1000000);
         reader.close();
         dir.close();
diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
index 0d83777dbd4cf..4c7770c563764 100644
--- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
@@ -99,27 +99,35 @@ private static QuerySearchResult createTestInstance() throws Exception {
         if (randomBoolean()) {
             result.aggregations(InternalAggregationsTests.createTestInstance());
         }
-        assertEquals(0, result.getTookTimeNanos());
+        assertNull(result.getTookTimeNanos());
         return result;
     }
 
     public void testSerialization() throws Exception {
-        QuerySearchResult querySearchResult = createTestInstance();
-        QuerySearchResult deserialized = copyWriteable(querySearchResult, namedWriteableRegistry, QuerySearchResult::new);
-        assertEquals(querySearchResult.getContextId().getId(), deserialized.getContextId().getId());
-        assertNull(deserialized.getSearchShardTarget());
-        assertEquals(querySearchResult.topDocs().maxScore, deserialized.topDocs().maxScore, 0f);
-        assertEquals(querySearchResult.topDocs().topDocs.totalHits, deserialized.topDocs().topDocs.totalHits);
-        assertEquals(querySearchResult.from(), deserialized.from());
-        assertEquals(querySearchResult.size(), deserialized.size());
-        assertEquals(querySearchResult.hasAggs(), deserialized.hasAggs());
-        if (deserialized.hasAggs()) {
-            Aggregations aggs = querySearchResult.consumeAggs().expand();
-            Aggregations deserializedAggs = deserialized.consumeAggs().expand();
-            assertEquals(aggs.asList(), deserializedAggs.asList());
+        for (int i = 0; i < 2; i++) {
+            QuerySearchResult querySearchResult = createTestInstance();
+            if (i == 0) {
+                querySearchResult.setTookTimeNanos(1000L); // test both an initialized and an uninitialized null value
+            }
+            QuerySearchResult deserialized = copyWriteable(querySearchResult, namedWriteableRegistry, QuerySearchResult::new);
+            assertEquals(querySearchResult.getContextId().getId(), deserialized.getContextId().getId());
+            assertNull(deserialized.getSearchShardTarget());
+            assertEquals(querySearchResult.topDocs().maxScore, deserialized.topDocs().maxScore, 0f);
+            assertEquals(querySearchResult.topDocs().topDocs.totalHits, deserialized.topDocs().topDocs.totalHits);
+            assertEquals(querySearchResult.from(), deserialized.from());
+            assertEquals(querySearchResult.size(), deserialized.size());
+            assertEquals(querySearchResult.hasAggs(), deserialized.hasAggs());
+            if (deserialized.hasAggs()) {
+                Aggregations aggs = querySearchResult.consumeAggs().expand();
+                Aggregations deserializedAggs = deserialized.consumeAggs().expand();
+                assertEquals(aggs.asList(), deserializedAggs.asList());
+            }
+            assertEquals(querySearchResult.terminatedEarly(), deserialized.terminatedEarly());
+            assertEquals(querySearchResult.getTookTimeNanos(), deserialized.getTookTimeNanos());
+            if (i == 1) {
+                assertNull(deserialized.getTookTimeNanos());
+            }
         }
-        assertEquals(querySearchResult.terminatedEarly(), deserialized.terminatedEarly());
-        assertEquals(querySearchResult.getTookTimeNanos(), deserialized.getTookTimeNanos());
     }
 
     public void testNullResponse() throws Exception {

From 1d47f38e5d7d485a21f2877cf80ec080bf0237ee Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 31 Oct 2023 11:51:20 -0700
Subject: [PATCH 040/120] Addressed dblock's comment

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../search/query/QuerySearchResultTests.java      | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
index 4c7770c563764..1b8fc9d7dbc5c 100644
--- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java
@@ -56,6 +56,8 @@
 import org.opensearch.search.suggest.SuggestTests;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.util.HashMap;
+
 import static java.util.Collections.emptyList;
 
 public class QuerySearchResultTests extends OpenSearchTestCase {
@@ -104,10 +106,13 @@ private static QuerySearchResult createTestInstance() throws Exception {
     }
 
     public void testSerialization() throws Exception {
-        for (int i = 0; i < 2; i++) {
+        HashMap<Boolean, Long> expectedValues = new HashMap<>(); // map contains whether to set took time, and if so, to what value
+        expectedValues.put(false, null);
+        expectedValues.put(true, 1000L);
+        for (Boolean doSetTookTime : expectedValues.keySet()) {
             QuerySearchResult querySearchResult = createTestInstance();
-            if (i == 0) {
-                querySearchResult.setTookTimeNanos(1000L); // test both an initialized and an uninitialized null value
+            if (doSetTookTime) {
+                querySearchResult.setTookTimeNanos(expectedValues.get(doSetTookTime));
             }
             QuerySearchResult deserialized = copyWriteable(querySearchResult, namedWriteableRegistry, QuerySearchResult::new);
             assertEquals(querySearchResult.getContextId().getId(), deserialized.getContextId().getId());
@@ -124,9 +129,7 @@ public void testSerialization() throws Exception {
             }
             assertEquals(querySearchResult.terminatedEarly(), deserialized.terminatedEarly());
             assertEquals(querySearchResult.getTookTimeNanos(), deserialized.getTookTimeNanos());
-            if (i == 1) {
-                assertNull(deserialized.getTookTimeNanos());
-            }
+            assertEquals(expectedValues.get(doSetTookTime), querySearchResult.getTookTimeNanos());
         }
     }
 

From e39514b8a6c4da99a72201ad85ab51556f84f73f Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 1 Nov 2023 16:59:52 -0700
Subject: [PATCH 041/120] Adds Serializer interface for use in ehcache disk
 tier and elsewhere; modifies existing disk tier impl to use it in a generic
 way

Signed-off-by: Peter Alfonsi <petealft@amazon.com>

Fixed byte[] key implementation to use ByteBuffer wrapper passed directly to ehcache

Signed-off-by: Peter Alfonsi <petealft@amazon.com>

Added tests for BytesReference serializer, and ehcache disk tier using BytesReference as a value
---
 .../cache/tier/BytesReferenceSerializer.java  |  35 +++++
 .../cache/tier/EhCacheDiskCachingTier.java    | 128 ++++++++++++++----
 .../common/cache/tier/Serializer.java         |  39 ++++++
 .../tier/BytesReferenceSerializerTests.java   |  61 +++++++++
 .../tier/EhCacheDiskCachingTierTests.java     |  70 ++++++++++
 5 files changed, 309 insertions(+), 24 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/Serializer.java
 create mode 100644 server/src/test/java/org/opensearch/common/cache/tier/BytesReferenceSerializerTests.java

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java b/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
new file mode 100644
index 0000000000000..55ffe22c2a339
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class BytesReferenceSerializer implements Serializer<BytesReference, byte[]> {
+    // This class does not get passed to ehcache itself, so it's not required that classes match after deserialization.
+
+    public BytesReferenceSerializer() {}
+    @Override
+    public byte[] serialize(BytesReference object) {
+        return BytesReference.toBytes(object);
+    }
+
+    @Override
+    public BytesReference deserialize(byte[] bytes) {
+        return new BytesArray(bytes);
+    }
+
+    @Override
+    public boolean equals(BytesReference object, byte[] bytes) {
+        return Arrays.equals(serialize(object), bytes);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index e7e52c7fbccbf..4adfcd731c6ef 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.common.cache.tier;
 
+import org.ehcache.core.spi.service.FileBasedPersistenceContext;
+import org.ehcache.spi.serialization.SerializerException;
 import org.opensearch.OpenSearchException;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
@@ -18,6 +20,7 @@
 import org.opensearch.common.unit.TimeValue;
 
 import java.io.File;
+import java.nio.ByteBuffer;
 import java.time.Duration;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
@@ -40,13 +43,25 @@
 import org.ehcache.expiry.ExpiryPolicy;
 import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
 
+/**
+ * This ehcache disk caching tier uses its value serializer outside ehcache.
+ * Values are transformed to byte[] outside ehcache and then ehcache uses its bundled byte[] serializer.
+ * The key serializer you pass to this class produces a byte[]. This serializer is passed to a wrapper which
+ * implements Ehcache's serializer implementation and produces a BytesBuffer. The wrapper instance is then passed to ehcache.
+ * This is done because to get keys on a disk tier, ehcache internally checks the equals() method of the serializer,
+ * but ALSO requires newKey.equals(storedKey) (this isn't documented), which is the case for ByteBuffer but not byte[].
+ * This limitation means that the key serializer must preserve the class of the key before/after serialization,
+ * but the value serializer does not have to do this.
+ * @param <K> The key type of cache entries
+ * @param <V> The value type of cache entries
+ */
 public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
 
     // A Cache manager can create many caches.
     private final PersistentCacheManager cacheManager;
 
     // Disk cache
-    private Cache<K, V> cache;
+    private Cache<K, byte[]> cache;
     private final long maxWeightInBytes;
     private final String storagePath;
 
@@ -86,11 +101,16 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
     // will hold that many file pointers.
     public final Setting<Integer> DISK_SEGMENTS;
 
+    private final Serializer<K, byte[]> keySerializer;
+    private final Serializer<V, byte[]> valueSerializer;
+
     private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.keyType = Objects.requireNonNull(builder.keyType, "Key type shouldn't be null");
         this.valueType = Objects.requireNonNull(builder.valueType, "Value type shouldn't be null");
         this.expireAfterAccess = Objects.requireNonNull(builder.expireAfterAcess, "ExpireAfterAccess value shouldn't " + "be null");
-        this.ehCacheEventListener = new EhCacheEventListener<K, V>();
+        this.keySerializer = Objects.requireNonNull(builder.keySerializer, "Key serializer shouldn't be null");
+        this.valueSerializer = Objects.requireNonNull(builder.valueSerializer, "Value serializer shouldn't be null");
+        this.ehCacheEventListener = new EhCacheEventListener<K, V>(this.valueSerializer);
         this.maxWeightInBytes = builder.maxWeightInBytes;
         this.storagePath = Objects.requireNonNull(builder.storagePath, "Storage path shouldn't be null");
         if (builder.threadPoolAlias == null || builder.threadPoolAlias.isBlank()) {
@@ -124,37 +144,38 @@ private PersistentCacheManager buildCacheManager() {
             .build(true);
     }
 
-    private Cache<K, V> buildCache(Duration expireAfterAccess, Builder<K, V> builder) {
+    private Cache<K, byte[]> buildCache(Duration expireAfterAccess, Builder<K, V> builder) {
         return this.cacheManager.createCache(
             DISK_CACHE_ALIAS,
             CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                this.keyType,
-                this.valueType,
+                keyType,
+                byte[].class,
                 ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B)
-            ).withExpiry(new ExpiryPolicy<K, V>() {
+            ).withExpiry(new ExpiryPolicy<K, byte[]>() {
                 @Override
-                public Duration getExpiryForCreation(K key, V value) {
+                public Duration getExpiryForCreation(K key, byte[] value) {
                     return INFINITE;
                 }
 
                 @Override
-                public Duration getExpiryForAccess(K key, Supplier<? extends V> value) {
+                public Duration getExpiryForAccess(K key, Supplier<? extends byte[]> value) {
                     return expireAfterAccess;
                 }
 
                 @Override
-                public Duration getExpiryForUpdate(K key, Supplier<? extends V> oldValue, V newValue) {
+                public Duration getExpiryForUpdate(K key, Supplier<? extends byte[]> oldValue, byte[] newValue) {
                     return INFINITE;
                 }
             })
                 .withService(getListenerConfiguration(builder))
                 .withService(
-                    new OffHeapDiskStoreConfiguration(
-                        this.threadPoolAlias,
-                        DISK_WRITE_CONCURRENCY.get(settings),
-                        DISK_SEGMENTS.get(settings)
-                    )
+                new OffHeapDiskStoreConfiguration(
+                    this.threadPoolAlias,
+                    DISK_WRITE_CONCURRENCY.get(settings),
+                    DISK_SEGMENTS.get(settings)
                 )
+                )
+                .withKeySerializer(new KeySerializerWrapper<K>(keySerializer))
         );
     }
 
@@ -177,12 +198,12 @@ private CacheEventListenerConfigurationBuilder getListenerConfiguration(Builder<
     @Override
     public V get(K key) {
         // Optimize it by adding key store.
-        return cache.get(key);
+        return valueSerializer.deserialize(cache.get(key));
     }
 
     @Override
     public void put(K key, V value) {
-        cache.put(key, value);
+        cache.put(key, valueSerializer.serialize(value));
     }
 
     @Override
@@ -193,7 +214,7 @@ public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception
 
     @Override
     public void invalidate(K key) {
-        // There seems to be an thread leak issue while calling this and then closing cache.
+        // There seems to be a thread leak issue while calling this and then closing cache.
         cache.remove(key);
     }
 
@@ -244,18 +265,23 @@ public void close() {
      * @param <K> Type of key
      * @param <V> Type of value
      */
-    class EhCacheEventListener<K, V> implements CacheEventListener<K, V> {
+    class EhCacheEventListener<K, V> implements CacheEventListener<K, byte[]> {
 
         private Optional<RemovalListener<K, V>> removalListener;
+        // We need to pass the value serializer to this listener, as the removal listener is expecting
+        // values of type K, V, not K, byte[]
+        private Serializer<V, byte[]> valueSerializer;
 
-        EhCacheEventListener() {}
+        EhCacheEventListener(Serializer<V, byte[]> valueSerializer) {
+            this.valueSerializer = valueSerializer;
+        }
 
         public void setRemovalListener(RemovalListener<K, V> removalListener) {
             this.removalListener = Optional.ofNullable(removalListener);
         }
 
         @Override
-        public void onEvent(CacheEvent<? extends K, ? extends V> event) {
+        public void onEvent(CacheEvent<? extends K, ? extends byte[]> event) {
             switch (event.getType()) {
                 case CREATED:
                     count.inc();
@@ -264,7 +290,11 @@ public void onEvent(CacheEvent<? extends K, ? extends V> event) {
                 case EVICTED:
                     this.removalListener.ifPresent(
                         listener -> listener.onRemoval(
-                            new RemovalNotification<>(event.getKey(), event.getOldValue(), RemovalReason.EVICTED)
+                            new RemovalNotification<>(
+                                event.getKey(),
+                                valueSerializer.deserialize(event.getOldValue()),
+                                RemovalReason.EVICTED
+                            )
                         )
                     );
                     count.dec();
@@ -277,7 +307,11 @@ public void onEvent(CacheEvent<? extends K, ? extends V> event) {
                 case EXPIRED:
                     this.removalListener.ifPresent(
                         listener -> listener.onRemoval(
-                            new RemovalNotification<>(event.getKey(), event.getOldValue(), RemovalReason.INVALIDATED)
+                            new RemovalNotification<>(
+                                event.getKey(),
+                                valueSerializer.deserialize(event.getOldValue()),
+                                RemovalReason.INVALIDATED
+                            )
                         )
                     );
                     count.dec();
@@ -297,9 +331,9 @@ public void onEvent(CacheEvent<? extends K, ? extends V> event) {
      */
     class EhCacheKeyIterator<K> implements Iterator<K> {
 
-        Iterator<Cache.Entry<K, V>> iterator;
+        Iterator<Cache.Entry<K, byte[]>> iterator;
 
-        EhCacheKeyIterator(Iterator<Cache.Entry<K, V>> iterator) {
+        EhCacheKeyIterator(Iterator<Cache.Entry<K, byte[]>> iterator) {
             this.iterator = iterator;
         }
 
@@ -317,6 +351,40 @@ public K next() {
         }
     }
 
+    /**
+     * The wrapper for the key serializer which is passed directly to Ehcache.
+     */
+    private class KeySerializerWrapper<K> implements org.ehcache.spi.serialization.Serializer<K> {
+        public Serializer<K, byte[]> serializer;
+        public KeySerializerWrapper(Serializer<K, byte[]> serializer) {
+            this.serializer = serializer;
+        }
+
+        // This constructor must be present, but does not have to work as we are not actually persisting the disk
+        // cache after a restart.
+        // See https://www.ehcache.org/documentation/3.0/serializers-copiers.html#persistent-vs-transient-caches
+        public KeySerializerWrapper(ClassLoader classLoader, FileBasedPersistenceContext persistenceContext) {}
+
+        @Override
+        public ByteBuffer serialize(K object) throws SerializerException {
+            return ByteBuffer.wrap(serializer.serialize(object));
+        }
+
+        @Override
+        public K read(ByteBuffer binary) throws ClassNotFoundException, SerializerException {
+            byte[] arr = new byte[binary.remaining()];
+            binary.get(arr);
+            return serializer.deserialize(arr);
+        }
+
+        @Override
+        public boolean equals(K object, ByteBuffer binary) throws ClassNotFoundException, SerializerException {
+            byte[] arr = new byte[binary.remaining()];
+            binary.get(arr);
+            return serializer.equals(object, arr);
+        }
+    }
+
     /**
      * Builder object to build Ehcache disk tier.
      * @param <K> Type of key
@@ -342,6 +410,8 @@ public static class Builder<K, V> {
 
         // Provides capability to make ehCache event listener to run in sync mode. Used for testing too.
         private boolean isEventListenerModeSync;
+        private Serializer<K, byte[]> keySerializer;
+        private Serializer<V, byte[]> valueSerializer;
 
         public Builder() {}
 
@@ -399,6 +469,16 @@ public EhCacheDiskCachingTier.Builder<K, V> setIsEventListenerModeSync(boolean i
             return this;
         }
 
+        public EhCacheDiskCachingTier.Builder<K, V> setKeySerializer(Serializer<K, byte[]> keySerializer) {
+            this.keySerializer = keySerializer;
+            return this;
+        }
+
+        public EhCacheDiskCachingTier.Builder<K, V> setValueSerializer(Serializer<V, byte[]> valueSerializer) {
+            this.valueSerializer = valueSerializer;
+            return this;
+        }
+
         public EhCacheDiskCachingTier<K, V> build() {
             return new EhCacheDiskCachingTier<>(this);
         }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/Serializer.java b/server/src/main/java/org/opensearch/common/cache/tier/Serializer.java
new file mode 100644
index 0000000000000..74c256d188682
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/Serializer.java
@@ -0,0 +1,39 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+import java.io.IOException;
+
+/**
+ * An interface for serializers, to be used in disk caching tier and elsewhere.
+ * T is the class of the original object, and U is the serialized class.
+ */
+public interface Serializer<T, U> {
+    /**
+     * Serializes an object.
+     * @param object A non-serialized object.
+     * @return The serialized representation of the object.
+     */
+    U serialize(T object);
+
+    /**
+     * Deserializes bytes into an object.
+     * @param bytes The serialized representation.
+     * @return The original object.
+     */
+    T deserialize(U bytes);
+
+    /**
+     * Compares an object to a serialized representation of an object.
+     * @param object A non-serialized objet
+     * @param bytes Serialized representation of an object
+     * @return true if representing the same object, false if not
+     */
+    boolean equals(T object, U bytes);
+}
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/BytesReferenceSerializerTests.java b/server/src/test/java/org/opensearch/common/cache/tier/BytesReferenceSerializerTests.java
new file mode 100644
index 0000000000000..2fc9c7cbb2756
--- /dev/null
+++ b/server/src/test/java/org/opensearch/common/cache/tier/BytesReferenceSerializerTests.java
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+import org.opensearch.common.Randomness;
+import org.opensearch.common.bytes.ReleasableBytesReference;
+import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.PageCacheRecycler;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.bytes.CompositeBytesReference;
+import org.opensearch.core.common.util.ByteArray;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.Random;
+
+public class BytesReferenceSerializerTests extends OpenSearchTestCase {
+    public void testEquality() throws Exception {
+        BytesReferenceSerializer ser = new BytesReferenceSerializer();
+        // Test that values are equal before and after serialization, for each implementation of BytesReference.
+        byte[] bytesValue = new byte[1000];
+        Random rand = Randomness.get();
+        rand.nextBytes(bytesValue);
+
+        BytesReference ba = new BytesArray(bytesValue);
+        byte[] serialized = ser.serialize(ba);
+        assertTrue(ser.equals(ba, serialized));
+        BytesReference deserialized = ser.deserialize(serialized);
+        assertEquals(ba, deserialized);
+
+        BytesReference cbr = CompositeBytesReference.of(new BytesArray(bytesValue), new BytesArray(bytesValue));
+        serialized = ser.serialize(cbr);
+        assertTrue(ser.equals(cbr, serialized));
+        deserialized = ser.deserialize(serialized);
+        assertEquals(cbr, deserialized);
+
+        // We need the PagedBytesReference to be larger than the page size (16 KB) in order to actually create it
+        byte[] pbrValue = new byte[PageCacheRecycler.PAGE_SIZE_IN_BYTES * 2];
+        rand.nextBytes(pbrValue);
+        ByteArray arr = BigArrays.NON_RECYCLING_INSTANCE.newByteArray(pbrValue.length);
+        arr.set(0L, pbrValue, 0, pbrValue.length);
+        assert !arr.hasArray();
+        BytesReference pbr = BytesReference.fromByteArray(arr, pbrValue.length);
+        serialized = ser.serialize(pbr);
+        assertTrue(ser.equals(pbr, serialized));
+        deserialized = ser.deserialize(serialized);
+        assertEquals(pbr, deserialized);
+
+        BytesReference rbr = new ReleasableBytesReference(new BytesArray(bytesValue), ReleasableBytesReference.NO_OP);
+        serialized = ser.serialize(rbr);
+        assertTrue(ser.equals(rbr, serialized));
+        deserialized = ser.deserialize(serialized);
+        assertEquals(rbr, deserialized);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
index 804f236264daa..c873fc075c4cc 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
@@ -8,16 +8,22 @@
 
 package org.opensearch.common.cache.tier;
 
+import org.opensearch.common.Randomness;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.env.NodeEnvironment;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 
 import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Phaser;
@@ -40,6 +46,8 @@ public void testBasicGetAndPut() throws IOException {
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
                 .setSettingPrefix(SETTING_PREFIX)
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
                 .build();
             int randomKeys = randomIntBetween(10, 100);
             Map<String, String> keyValueMap = new HashMap<>();
@@ -57,6 +65,41 @@ public void testBasicGetAndPut() throws IOException {
         }
     }
 
+    public void testBasicGetAndPutBytesReference() throws Exception {
+        Settings settings = Settings.builder().build();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            EhCacheDiskCachingTier<String, BytesReference> ehCacheDiskCachingTier = new EhCacheDiskCachingTier.Builder<String, BytesReference>()
+                .setKeyType(String.class)
+                .setValueType(BytesReference.class)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setSettings(settings)
+                .setThreadPoolAlias("ehcacheTest")
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
+                .setSettingPrefix(SETTING_PREFIX)
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new BytesReferenceSerializer())
+                .build();
+            int randomKeys = randomIntBetween(10, 100);
+            int valueLength = 1000;
+            Random rand = Randomness.get();
+            Map<String, BytesReference> keyValueMap = new HashMap<>();
+            for (int i = 0; i < randomKeys; i++) {
+                byte[] valueBytes = new byte[valueLength];
+                rand.nextBytes(valueBytes);
+                keyValueMap.put(UUID.randomUUID().toString(), new BytesArray(valueBytes));
+            }
+            for (Map.Entry<String, BytesReference> entry : keyValueMap.entrySet()) {
+                ehCacheDiskCachingTier.put(entry.getKey(), entry.getValue());
+            }
+            for (Map.Entry<String, BytesReference> entry : keyValueMap.entrySet()) {
+                BytesReference value = ehCacheDiskCachingTier.get(entry.getKey());
+                assertEquals(entry.getValue(), value);
+            }
+            ehCacheDiskCachingTier.close();
+        }
+    }
+
     public void testConcurrentPut() throws Exception {
         Settings settings = Settings.builder().build();
         try (NodeEnvironment env = newNodeEnvironment(settings)) {
@@ -69,6 +112,8 @@ public void testConcurrentPut() throws Exception {
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
                 .setSettingPrefix(SETTING_PREFIX)
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
                 .build();
             int randomKeys = randomIntBetween(20, 100);
             Thread[] threads = new Thread[randomKeys];
@@ -111,6 +156,8 @@ public void testEhcacheParallelGets() throws Exception {
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
                 .setSettingPrefix(SETTING_PREFIX)
                 .setIsEventListenerModeSync(true) // For accurate count
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
                 .build();
             ehCacheDiskCachingTierNew.setRemovalListener(removalListener(new AtomicInteger()));
             int randomKeys = randomIntBetween(20, 100);
@@ -153,6 +200,8 @@ public void testEhcacheKeyIterator() throws Exception {
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setSettingPrefix(SETTING_PREFIX)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
                 .build();
 
             int randomKeys = randomIntBetween(2, 2);
@@ -187,6 +236,8 @@ public void testCompute() throws Exception {
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setSettingPrefix(SETTING_PREFIX)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
                 .build();
             // For now it is unsupported.
             assertThrows(
@@ -223,4 +274,23 @@ public boolean isLoaded() {
     private RemovalListener<String, String> removalListener(AtomicInteger counter) {
         return notification -> counter.incrementAndGet();
     }
+
+    private static class StringSerializer implements Serializer<String, byte[]> {
+
+        private final Charset charset = StandardCharsets.UTF_8;
+        @Override
+        public byte[] serialize(String object) {
+            return object.getBytes(charset);
+        }
+
+        @Override
+        public String deserialize(byte[] bytes) {
+            return new String(bytes, charset);
+        }
+
+        @Override
+        public boolean equals(String object, byte[] bytes) {
+            return object.equals(deserialize(bytes));
+        }
+    }
 }

From d45e6cf2e5cf22c5800da32276e504267f8233c0 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 2 Nov 2023 16:30:20 -0700
Subject: [PATCH 042/120] Addressed Sagar's comments besides new
 counter/arraylist setup for removing keys

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../cache/tier/keystore}/KeyLookupStore.java  |  5 +-
 .../cache/tier/keystore/KeyStoreStats.java    | 40 ++++++++
 .../tier/keystore}/RBMIntKeyLookupStore.java  | 98 ++++++++++---------
 .../tier/keystore}/RBMSizeEstimator.java      | 10 +-
 .../keystore}/RBMIntKeyLookupStoreTests.java  | 67 +++++++------
 5 files changed, 137 insertions(+), 83 deletions(-)
 rename server/src/main/java/org/opensearch/{indices => common/cache/tier/keystore}/KeyLookupStore.java (98%)
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java
 rename server/src/main/java/org/opensearch/{indices => common/cache/tier/keystore}/RBMIntKeyLookupStore.java (77%)
 rename server/src/main/java/org/opensearch/{indices => common/cache/tier/keystore}/RBMSizeEstimator.java (92%)
 rename server/src/test/java/org/opensearch/{indices => common/cache/tier/keystore}/RBMIntKeyLookupStoreTests.java (80%)

diff --git a/server/src/main/java/org/opensearch/indices/KeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
similarity index 98%
rename from server/src/main/java/org/opensearch/indices/KeyLookupStore.java
rename to server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
index 60e1386a460ec..552698e0293eb 100644
--- a/server/src/main/java/org/opensearch/indices/KeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
@@ -30,7 +30,7 @@
  * GitHub history for details.
  */
 
-package org.opensearch.indices;
+package org.opensearch.common.cache.tier.keystore;
 
 /**
  * An interface for objects that hold an in-memory record of hashes of keys in the disk cache.
@@ -85,7 +85,7 @@ public interface KeyLookupStore<T> {
      * Returns the number of times add() has been run, including unsuccessful attempts.
      * @return The number of adding attempts.
      */
-    int getTotalAdds();
+    int getAddAttempts();
 
     /**
      * Returns the number of times add() has returned false due to a collision.
@@ -93,7 +93,6 @@ public interface KeyLookupStore<T> {
      */
     int getCollisions();
 
-
     /**
      * Checks if two values would collide after being transformed by this store's transformation.
      * @param value1 The first value to compare.
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java
new file mode 100644
index 0000000000000..b5c95b134990c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier.keystore;
+
+import org.opensearch.common.metrics.CounterMetric;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A stats holder for use in KeyLookupStore implementations.
+ * Getters should be exposed by the KeyLookupStore which uses it.
+ */
+public class KeyStoreStats {
+    protected CounterMetric size;
+    protected long memSizeCapInBytes;
+    protected CounterMetric numAddAttempts;
+    protected CounterMetric numCollisions;
+    protected boolean guaranteesNoFalseNegatives;
+    protected final int maxNumEntries;
+    protected AtomicBoolean atCapacity;
+    protected CounterMetric numRemovalAttempts;
+    protected CounterMetric numSuccessfulRemovals;
+
+    protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) {
+        this.size = new CounterMetric();
+        this.numAddAttempts = new CounterMetric();
+        this.numCollisions = new CounterMetric();
+        this.memSizeCapInBytes = memSizeCapInBytes;
+        this.maxNumEntries = maxNumEntries;
+        this.atCapacity = new AtomicBoolean(false);
+        this.numRemovalAttempts = new CounterMetric();
+        this.numSuccessfulRemovals = new CounterMetric();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
similarity index 77%
rename from server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java
rename to server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index 3789989b5eaf1..7e91a64758919 100644
--- a/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -30,15 +30,16 @@
  * GitHub history for details.
  */
 
-package org.opensearch.indices;
+package org.opensearch.common.cache.tier.keystore;
 
 import org.opensearch.common.metrics.CounterMetric;
-import org.roaringbitmap.RoaringBitmap;
 
 import java.util.HashSet;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import org.roaringbitmap.RoaringBitmap;
+
 /**
  * This class implements KeyLookupStore<Integer> using a roaring bitmap with a modulo applied to values.
  * The modulo increases the density of values, which makes RBMs more memory-efficient. The recommended modulo is ~2^28.
@@ -48,30 +49,29 @@
  * The store estimates its memory footprint and will stop adding more values once it reaches its memory cap.
  */
 public class RBMIntKeyLookupStore implements KeyLookupStore<Integer> {
-    protected final int modulo;
-    protected class KeyStoreStats {
-        protected int size;
-        protected long memSizeCapInBytes;
-        protected CounterMetric numAddAttempts;
-        protected CounterMetric numCollisions;
-        protected boolean guaranteesNoFalseNegatives;
-        protected int maxNumEntries;
-        protected boolean atCapacity;
-        protected CounterMetric numRemovalAttempts;
-        protected CounterMetric numSuccessfulRemovals;
-        protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) {
-            this.size = 0;
-            this.numAddAttempts = new CounterMetric();
-            this.numCollisions = new CounterMetric();
-            this.memSizeCapInBytes = memSizeCapInBytes;
-            this.maxNumEntries = maxNumEntries;
-            this.atCapacity = false;
-            this.numRemovalAttempts = new CounterMetric();
-            this.numSuccessfulRemovals = new CounterMetric();
+    /**
+     * An enum representing modulo values for use in the keystore
+     */
+    public enum KeystoreModuloValue {
+        NONE(0), // No modulo applied
+        TWO_TO_THIRTY_ONE((int) Math.pow(2, 31)),
+        TWO_TO_TWENTY_NINE((int) Math.pow(2, 29)), // recommended value
+        TWO_TO_TWENTY_EIGHT((int) Math.pow(2, 28)),
+        TWO_TO_TWENTY_SIX((int) Math.pow(2, 26));
+
+        private final int value;
+
+        private KeystoreModuloValue(int value) {
+            this.value = value;
+        }
+
+        public int getValue() {
+            return this.value;
         }
     }
 
-    protected KeyStoreStats stats;
+    protected final int modulo;
+    KeyStoreStats stats;
     protected RoaringBitmap rbm;
     private HashSet<Integer> collidedInts;
     protected RBMSizeEstimator sizeEstimator;
@@ -79,8 +79,13 @@ protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) {
     protected final Lock readLock = lock.readLock();
     protected final Lock writeLock = lock.writeLock();
 
-    RBMIntKeyLookupStore(int modulo, long memSizeCapInBytes) {
-        this.modulo = modulo;
+    // Default constructor sets modulo = 2^28
+    public RBMIntKeyLookupStore(long memSizeCapInBytes) {
+        this(KeystoreModuloValue.TWO_TO_TWENTY_EIGHT, memSizeCapInBytes);
+    }
+
+    public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBytes) {
+        this.modulo = moduloValue.getValue();
         sizeEstimator = new RBMSizeEstimator(modulo);
         this.stats = new KeyStoreStats(memSizeCapInBytes, calculateMaxNumEntries(memSizeCapInBytes));
         this.rbm = new RoaringBitmap();
@@ -94,11 +99,11 @@ protected int calculateMaxNumEntries(long memSizeCapInBytes) {
         return sizeEstimator.getNumEntriesFromSizeInBytes(memSizeCapInBytes);
     }
 
-    protected final int transform(int value) {
+    private final int transform(int value) {
         return modulo == 0 ? value : value % modulo;
     }
 
-    protected void handleCollisions(int transformedValue) {
+    private void handleCollisions(int transformedValue) {
         stats.numCollisions.inc();
         collidedInts.add(transformedValue);
     }
@@ -108,18 +113,21 @@ public boolean add(Integer value) throws Exception {
         if (value == null) {
             return false;
         }
-        writeLock.lock();
         stats.numAddAttempts.inc();
+        if (stats.size.count() == stats.maxNumEntries) {
+            stats.atCapacity.set(true);
+            return false;
+        }
+        int transformedValue = transform(value);
+
+        writeLock.lock();
         try {
-            if (stats.size == stats.maxNumEntries) {
-                stats.atCapacity = true;
-                return false;
-            }
-            int transformedValue = transform(value);
-            boolean alreadyContained = contains(value);
+            boolean alreadyContained;
+            // saves calling transform() an additional time
+            alreadyContained = rbm.contains(transformedValue);
             if (!alreadyContained) {
                 rbm.add(transformedValue);
-                stats.size++;
+                stats.size.inc();
                 return true;
             }
             handleCollisions(transformedValue);
@@ -159,7 +167,7 @@ public boolean remove(Integer value) throws Exception {
         int transformedValue = transform(value);
         readLock.lock();
         try {
-            if (!contains(value)) {
+            if (!rbm.contains(transformedValue)) { // saves additional transform() call
                 return false;
             }
             stats.numRemovalAttempts.inc();
@@ -172,7 +180,7 @@ public boolean remove(Integer value) throws Exception {
         writeLock.lock();
         try {
             rbm.remove(transformedValue);
-            stats.size--;
+            stats.size.dec();
             stats.numSuccessfulRemovals.inc();
             return true;
         } finally {
@@ -184,14 +192,14 @@ public boolean remove(Integer value) throws Exception {
     public int getSize() {
         readLock.lock();
         try {
-            return stats.size;
+            return (int) stats.size.count();
         } finally {
             readLock.unlock();
         }
     }
 
     @Override
-    public int getTotalAdds() {
+    public int getAddAttempts() {
         return (int) stats.numAddAttempts.count();
     }
 
@@ -200,7 +208,6 @@ public int getCollisions() {
         return (int) stats.numCollisions.count();
     }
 
-
     @Override
     public boolean isCollision(Integer value1, Integer value2) {
         if (value1 == null || value2 == null) {
@@ -211,7 +218,7 @@ public boolean isCollision(Integer value1, Integer value2) {
 
     @Override
     public long getMemorySizeInBytes() {
-        return sizeEstimator.getSizeInBytes(stats.size) + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size());
+        return sizeEstimator.getSizeInBytes((int) stats.size.count()) + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size());
     }
 
     @Override
@@ -221,14 +228,14 @@ public long getMemorySizeCapInBytes() {
 
     @Override
     public boolean isFull() {
-        return stats.atCapacity;
+        return stats.atCapacity.get();
     }
 
     @Override
     public void regenerateStore(Integer[] newValues) throws Exception {
         rbm.clear();
         collidedInts = new HashSet<>();
-        stats.size = 0;
+        stats.size = new CounterMetric();
         stats.numAddAttempts = new CounterMetric();
         stats.numCollisions = new CounterMetric();
         stats.guaranteesNoFalseNegatives = true;
@@ -241,12 +248,11 @@ public void regenerateStore(Integer[] newValues) throws Exception {
         }
     }
 
-
-
     @Override
     public void clear() throws Exception {
-        regenerateStore(new Integer[]{});
+        regenerateStore(new Integer[] {});
     }
+
     public int getNumRemovalAttempts() {
         return (int) stats.numRemovalAttempts.count();
     }
diff --git a/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java
similarity index 92%
rename from server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java
rename to server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java
index 6e3b8e581ba9f..e07c645bad0cf 100644
--- a/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java
@@ -30,7 +30,9 @@
  * GitHub history for details.
  */
 
-package org.opensearch.indices;
+package org.opensearch.common.cache.tier.keystore;
+
+import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
 
 /**
  * A class used to estimate roaring bitmap memory sizes (and hash set sizes).
@@ -111,12 +113,16 @@ public static long getSizeInBytesWithModulo(int numEntries, int modulo) {
         return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]);
     }
 
+    public static long getSizeInBytesWithModuloValue(int numEntries, RBMIntKeyLookupStore.KeystoreModuloValue moduloValue) {
+        double[] memValues = calculateMemoryCoefficients(moduloValue.getValue());
+        return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]);
+    }
+
     public static int getNumEntriesFromSizeInBytesWithModulo(long sizeInBytes, int modulo) {
         double[] memValues = calculateMemoryCoefficients(modulo);
         return (int) Math.pow(sizeInBytes / (memValues[0] * Math.pow(10, memValues[2])), 1 / memValues[1]);
     }
 
-
     protected static long convertMBToBytes(double valMB) {
         return (long) (valMB * BYTES_IN_MB);
     }
diff --git a/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
similarity index 80%
rename from server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java
rename to server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index c857c1ecab768..08ab08462373b 100644
--- a/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -29,9 +29,11 @@
  * GitHub history for details.
  */
 
-package org.opensearch.indices;
+package org.opensearch.common.cache.tier.keystore;
 
 import org.opensearch.common.Randomness;
+import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
+import org.opensearch.common.cache.tier.keystore.RBMSizeEstimator;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.ArrayList;
@@ -43,13 +45,15 @@
 public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase {
     public void testInit() {
         long memCap = 100 * RBMSizeEstimator.BYTES_IN_MB;
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), memCap);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(memCap);
         assertEquals(0, kls.getSize());
+        assertEquals(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_EIGHT.getValue(), kls.modulo);
         assertEquals(memCap, kls.getMemorySizeCapInBytes());
     }
+
     public void testTransformationLogic() throws Exception {
         int modulo = (int) Math.pow(2, 29);
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
         int offset = 3;
         for (int i = 0; i < 4; i++) { // after this we run into max value, but thats not a flaw with the class design
             int posValue = i * modulo + offset;
@@ -58,7 +62,7 @@ public void testTransformationLogic() throws Exception {
             kls.add(negValue);
         }
         assertEquals(2, kls.getSize());
-        int[] testVals = new int[]{0, 1, -1, -23495, 23058, modulo, -modulo, Integer.MAX_VALUE, Integer.MIN_VALUE};
+        int[] testVals = new int[] { 0, 1, -1, -23495, 23058, modulo, -modulo, Integer.MAX_VALUE, Integer.MIN_VALUE };
         for (int value : testVals) {
             assertTrue(kls.getInternalRepresentation(value) < modulo);
             assertTrue(kls.getInternalRepresentation(value) > -modulo);
@@ -66,7 +70,7 @@ public void testTransformationLogic() throws Exception {
     }
 
     public void testContains() throws Exception {
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
         for (int i = 0; i < 2000; i++) {
             kls.add(i);
             assertTrue(kls.contains(i));
@@ -74,25 +78,25 @@ public void testContains() throws Exception {
     }
 
     public void testAddingStatsGetters() throws Exception {
-        int modulo = (int) Math.pow(2, 15);
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, 0L);
+        RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX;
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L);
         kls.add(15);
         kls.add(-15);
-        assertEquals(2, kls.getTotalAdds());
+        assertEquals(2, kls.getAddAttempts());
         assertEquals(0, kls.getCollisions());
 
         int offset = 1;
         for (int i = 0; i < 10; i++) {
-            kls.add(i * modulo + offset);
+            kls.add(i * moduloValue.getValue() + offset);
         }
-        assertEquals(12, kls.getTotalAdds());
+        assertEquals(12, kls.getAddAttempts());
         assertEquals(9, kls.getCollisions());
     }
 
     public void testRegenerateStore() throws Exception {
         int numToAdd = 10000000;
         Random rand = Randomness.get();
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
         for (int i = 0; i < numToAdd; i++) {
             kls.add(i);
         }
@@ -110,7 +114,7 @@ public void testRegenerateStore() throws Exception {
     }
 
     public void testAddingDuplicates() throws Exception {
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0L);
         int numToAdd = 4820411;
         for (int i = 0; i < numToAdd; i++) {
             kls.add(i);
@@ -123,10 +127,10 @@ public void testAddingDuplicates() throws Exception {
     }
 
     public void testMemoryCapBlocksAdd() throws Exception {
-        int modulo = (int) Math.pow(2, 29);
-        for (int maxEntries: new int[]{2342000, 1000, 100000}) {
-            long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModulo(maxEntries, modulo);
-            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memSizeCapInBytes);
+        RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE;
+        for (int maxEntries : new int[] { 2342000, 1000, 100000 }) {
+            long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue);
+            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes);
             for (int j = 0; j < maxEntries + 1000; j++) {
                 kls.add(j);
             }
@@ -139,7 +143,7 @@ public void testMemoryCapBlocksAdd() throws Exception {
     public void testConcurrency() throws Exception {
         Random rand = Randomness.get();
         for (int j = 0; j < 5; j++) { // test with different numbers of threads
-            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
             int numThreads = rand.nextInt(50) + 1;
             ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(numThreads);
             // In this test we want to add the first 200K numbers and check they're all correctly there.
@@ -201,7 +205,7 @@ public void testConcurrency() throws Exception {
     public void testRemoveNoCollisions() throws Exception {
         long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB;
         int numToAdd = 195000;
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0, memCap);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, memCap);
         // there should be no collisions for sequential positive numbers up to modulo
         for (int i = 0; i < numToAdd; i++) {
             kls.add(i);
@@ -217,7 +221,7 @@ public void testRemoveNoCollisions() throws Exception {
     public void testRemoveWithCollisions() throws Exception {
         int modulo = (int) Math.pow(2, 26);
         long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB;
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memCap);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX, memCap);
         for (int i = 0; i < 10; i++) {
             kls.add(i);
             if (i % 2 == 1) {
@@ -260,31 +264,30 @@ public void testRemoveWithCollisions() throws Exception {
     }
 
     public void testNullInputs() throws Exception {
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
         assertFalse(kls.add(null));
         assertFalse(kls.contains(null));
         assertEquals(0, (int) kls.getInternalRepresentation(null));
         assertFalse(kls.remove(null));
         assertFalse(kls.isCollision(null, null));
-        assertEquals(0, kls.getTotalAdds());
-        Integer[] newVals = new Integer[]{1, 17, -2, null, -4, null};
+        assertEquals(0, kls.getAddAttempts());
+        Integer[] newVals = new Integer[] { 1, 17, -2, null, -4, null };
         kls.regenerateStore(newVals);
         assertEquals(4, kls.getSize());
     }
 
     public void testMemoryCapValueInitialization() {
-        double[] logModulos = new double[] { 0.0, 31.2, 30, 29, 28, 13 };
-        double[] expectedMultipliers = new double[] { 1.2, 1.2, 1.2, 1, 1, 1 };
-        double[] expectedSlopes = new double[] { 0.637, 0.637, 0.637, 0.619, 0.614, 0.629 };
-        double[] expectedIntercepts = new double[] { 3.091, 3.091, 3.091, 2.993, 2.905, 2.603 };
+        // double[] logModulos = new double[] { 0.0, 31.2, 30, 29, 28, 13 };
+        // 0, 31, 29, 28, 26
+        RBMIntKeyLookupStore.KeystoreModuloValue[] mods = RBMIntKeyLookupStore.KeystoreModuloValue.values();
+        double[] expectedMultipliers = new double[] { 1.2, 1.2, 1, 1, 1 };
+        double[] expectedSlopes = new double[] { 0.637, 0.637, 0.619, 0.614, 0.629 };
+        double[] expectedIntercepts = new double[] { 3.091, 3.091, 2.993, 2.905, 2.603 }; // check the numbers closer later
         long memSizeCapInBytes = (long) 100.0 * RBMSizeEstimator.BYTES_IN_MB;
         double delta = 0.01;
-        for (int i = 0; i < logModulos.length; i++) {
-            int modulo = 0;
-            if (logModulos[i] != 0) {
-                modulo = (int) Math.pow(2, logModulos[i]);
-            }
-            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memSizeCapInBytes);
+        for (int i = 0; i < mods.length; i++) {
+            RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = mods[i];
+            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes);
             assertEquals(kls.stats.memSizeCapInBytes, kls.getMemorySizeCapInBytes(), 1.0);
             assertEquals(expectedMultipliers[i], kls.sizeEstimator.bufferMultiplier, delta);
             assertEquals(expectedSlopes[i], kls.sizeEstimator.slope, delta);

From 157ca6e9557cc5864c45c930be97efa6f17ea269 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 6 Nov 2023 09:44:27 -0800
Subject: [PATCH 043/120] Addressed comments besides moving IT tests into
 IndicesRequestCacheIT.java

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../IndicesRequestCacheDiskTierIT.java        | 28 ++++++++++++++-----
 .../indices/IndicesRequestCacheIT.java        | 20 ++++++++-----
 .../cache/request/RequestCacheStats.java      | 18 ++++++------
 3 files changed, 42 insertions(+), 24 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index 47bda50ca788a..e25fb0112ca01 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -37,6 +37,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
@@ -49,8 +50,7 @@
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class IndicesRequestCacheDiskTierIT extends OpenSearchIntegTestCase {
     public void testDiskTierStats() throws Exception {
-        int heapSizeBytes = 1800; // enough to fit 2 queries, as each is 687 B
-        int requestSize = 687; // each request is 687 B
+        int heapSizeBytes = 4729;
         String node = internalCluster().startNode(
             Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
         );
@@ -68,21 +68,35 @@ public void testDiskTierStats() throws Exception {
         ensureSearchable("index");
         SearchResponse resp;
 
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
+        int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
+        System.out.println(requestSize);
+        assertTrue(heapSizeBytes > requestSize);
+        // If this fails, increase heapSizeBytes! We can't adjust it after getting the size of one query
+        // as the cache size setting is not dynamic
+
         int numOnDisk = 5;
         int numRequests = heapSizeBytes / requestSize + numOnDisk;
-        for (int i = 0; i < numRequests; i++) {
+        for (int i = 1; i < numRequests; i++) {
             resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
             assertSearchResponse(resp);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
-            System.out.println("request number " + i);
         }
-
-        System.out.println("Num requests = " + numRequests);
-
         // the first request, for "hello0", should have been evicted to the disk tier
         resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello0")).get();
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 1, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 1, numRequests, TierType.DISK, false);
     }
+
+    private long getCacheSizeBytes(Client client, String index, TierType tierType) {
+        RequestCacheStats requestCacheStats = client.admin()
+            .indices()
+            .prepareStats(index)
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+        return requestCacheStats.getMemorySizeInBytes(tierType);
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index a9b18c6f8364b..080b3017c4246 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -637,13 +637,19 @@ public void testProfileDisableCache() throws Exception {
 
     public void testCacheWithInvalidation() throws Exception {
         Client client = client();
-
-        Settings.Builder builder = Settings.builder()
-            .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
-
-        assertAcked(client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(builder).get());
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate("index")
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                )
+                .get()
+        );
         indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
         ensureSearchable("index");
         SearchResponse resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index df5e5fee73cb5..34d91be1bc2c6 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -52,18 +52,17 @@
  */
 public class RequestCacheStats implements Writeable, ToXContentFragment {
 
-    private Map<String, StatsHolder> map;
-
-    public RequestCacheStats() {
-        this.map = new HashMap<>();
-        for (TierType tierType : TierType.values()) {
-            map.put(tierType.getStringValue(), new StatsHolder());
+    private Map<String, StatsHolder> map = new HashMap<>(){{
+        for (TierType tierType : TierType.values())
+        {
+            put(tierType.getStringValue(), new StatsHolder());
             // Every possible tier type must have counters, even if they are disabled. Then the counters report 0
-        }
-    }
+        }}
+    };
+
+    public RequestCacheStats() {}
 
     public RequestCacheStats(StreamInput in) throws IOException {
-        this();
         if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
             this.map = in.readMap(StreamInput::readString, StatsHolder::new);
         } else {
@@ -78,7 +77,6 @@ public RequestCacheStats(StreamInput in) throws IOException {
 
     public RequestCacheStats(Map<TierType, StatsHolder> inputMap) {
         // Create a RequestCacheStats with multiple tiers' statistics
-        this();
         for (TierType tierType : inputMap.keySet()) {
             map.put(tierType.getStringValue(), inputMap.get(tierType));
         }

From 404561644ba73a8e4ab6b5133a5cb9678d9d0580 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 6 Nov 2023 13:57:34 -0800
Subject: [PATCH 044/120] Implemented/tested counter+removal list setup to
 allow more removals

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../tier/keystore/RBMIntKeyLookupStore.java   |  91 +++++++++++++--
 .../keystore/RBMIntKeyLookupStoreTests.java   | 108 +++++++++++++++++-
 2 files changed, 184 insertions(+), 15 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index 7e91a64758919..f9696e62a88b7 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -34,6 +34,8 @@
 
 import org.opensearch.common.metrics.CounterMetric;
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -73,7 +75,8 @@ public int getValue() {
     protected final int modulo;
     KeyStoreStats stats;
     protected RoaringBitmap rbm;
-    private HashSet<Integer> collidedInts;
+    private HashMap<Integer, CounterMetric> collidedIntCounters;
+    private HashMap<Integer, HashSet<Integer>> removalSets;
     protected RBMSizeEstimator sizeEstimator;
     protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
     protected final Lock readLock = lock.readLock();
@@ -89,7 +92,8 @@ public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBy
         sizeEstimator = new RBMSizeEstimator(modulo);
         this.stats = new KeyStoreStats(memSizeCapInBytes, calculateMaxNumEntries(memSizeCapInBytes));
         this.rbm = new RoaringBitmap();
-        collidedInts = new HashSet<>();
+        this.collidedIntCounters = new HashMap<>();
+        this.removalSets = new HashMap<>();
     }
 
     protected int calculateMaxNumEntries(long memSizeCapInBytes) {
@@ -105,7 +109,14 @@ private final int transform(int value) {
 
     private void handleCollisions(int transformedValue) {
         stats.numCollisions.inc();
-        collidedInts.add(transformedValue);
+        CounterMetric numCollisions = collidedIntCounters.get(transformedValue);
+        if (numCollisions == null) { // First time the transformedValue has had a collision
+            numCollisions = new CounterMetric();
+            numCollisions.inc(2);
+            collidedIntCounters.put(transformedValue, numCollisions); // Initialize the number of colliding keys to 2
+        } else {
+            numCollisions.inc();
+        }
     }
 
     @Override
@@ -130,6 +141,16 @@ public boolean add(Integer value) throws Exception {
                 stats.size.inc();
                 return true;
             }
+            // If the value is already pending removal, take it out of the removalList
+            HashSet<Integer> removalSet = removalSets.get(transformedValue);
+            if (removalSet != null) {
+                removalSet.remove(value);
+                // Don't increment the counter - this is handled by handleCollisions() later
+                if (removalSet.isEmpty()) {
+                    removalSets.remove(transformedValue);
+                }
+            }
+
             handleCollisions(transformedValue);
             return false;
         } finally {
@@ -159,6 +180,13 @@ public Integer getInternalRepresentation(Integer value) {
         return Integer.valueOf(transform(value));
     }
 
+    /**
+     * Attempts to remove a value from the keystore. WARNING: Removing keys which have not been added to the keystore
+     * may cause undefined behavior, including future false negatives!!
+     * @param value The value to attempt to remove.
+     * @return true if the value was removed, false otherwise
+     * @throws Exception
+     */
     @Override
     public boolean remove(Integer value) throws Exception {
         if (value == null) {
@@ -170,24 +198,54 @@ public boolean remove(Integer value) throws Exception {
             if (!rbm.contains(transformedValue)) { // saves additional transform() call
                 return false;
             }
+            // move below into write lock
             stats.numRemovalAttempts.inc();
-            if (collidedInts.contains(transformedValue)) {
-                return false;
-            }
         } finally {
             readLock.unlock();
         }
         writeLock.lock();
         try {
-            rbm.remove(transformedValue);
-            stats.size.dec();
-            stats.numSuccessfulRemovals.inc();
+            CounterMetric numCollisions = collidedIntCounters.get(transformedValue);
+            if (numCollisions != null) {
+                // This transformed value has had a collision before
+                HashSet<Integer> removalSet = removalSets.get(transformedValue);
+                if (removalSet == null) {
+                    // First time a removal has been attempted for this transformed value
+                    HashSet<Integer> newRemovalSet = new HashSet<>();
+                    newRemovalSet.add(value); // Add the key value, not the transformed value, to the list of attempted removals for this transformedValue
+                    removalSets.put(transformedValue, newRemovalSet);
+                    numCollisions.dec();
+                } else {
+                    if (removalSet.contains(value)) {
+                        return false; // We have already attempted to remove this value. Do nothing
+                    }
+                    removalSet.add(value);
+                    numCollisions.dec();
+                    // If numCollisions has reached zero, we can safely remove all values in removalList
+                    if (numCollisions.count() == 0) {
+                        removeFromRBM(transformedValue);
+                        collidedIntCounters.remove(transformedValue);
+                        removalSets.remove(transformedValue);
+                        return true;
+                    }
+                }
+                return false;
+            }
+            // Otherwise, there's not been a collision for this transformedValue, so we can safely remove
+            removeFromRBM(transformedValue);
             return true;
         } finally {
             writeLock.unlock();
         }
     }
 
+    // Helper fn for remove()
+    private void removeFromRBM(int transformedValue) {
+        rbm.remove(transformedValue);
+        stats.size.dec();
+        stats.numSuccessfulRemovals.inc();
+    }
+
     @Override
     public int getSize() {
         readLock.lock();
@@ -218,7 +276,7 @@ public boolean isCollision(Integer value1, Integer value2) {
 
     @Override
     public long getMemorySizeInBytes() {
-        return sizeEstimator.getSizeInBytes((int) stats.size.count()) + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size());
+        return sizeEstimator.getSizeInBytes((int) stats.size.count()); // + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size());
     }
 
     @Override
@@ -234,7 +292,8 @@ public boolean isFull() {
     @Override
     public void regenerateStore(Integer[] newValues) throws Exception {
         rbm.clear();
-        collidedInts = new HashSet<>();
+        collidedIntCounters = new HashMap<>();
+        removalSets = new HashMap<>();
         stats.size = new CounterMetric();
         stats.numAddAttempts = new CounterMetric();
         stats.numCollisions = new CounterMetric();
@@ -265,6 +324,14 @@ public boolean valueHasHadCollision(Integer value) {
         if (value == null) {
             return false;
         }
-        return collidedInts.contains(transform(value));
+        return collidedIntCounters.containsKey(transform(value));
+    }
+
+    CounterMetric getNumCollisionsForValue(int value) { // package private for testing
+        return collidedIntCounters.get(transform(value));
+    }
+
+    HashSet<Integer> getRemovalSetForValue(int value) {
+        return removalSets.get(transform(value));
     }
 }
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index 08ab08462373b..0e9d63a9731a5 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -34,9 +34,11 @@
 import org.opensearch.common.Randomness;
 import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
 import org.opensearch.common.cache.tier.keystore.RBMSizeEstimator;
+import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.Random;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -277,12 +279,10 @@ public void testNullInputs() throws Exception {
     }
 
     public void testMemoryCapValueInitialization() {
-        // double[] logModulos = new double[] { 0.0, 31.2, 30, 29, 28, 13 };
-        // 0, 31, 29, 28, 26
         RBMIntKeyLookupStore.KeystoreModuloValue[] mods = RBMIntKeyLookupStore.KeystoreModuloValue.values();
         double[] expectedMultipliers = new double[] { 1.2, 1.2, 1, 1, 1 };
         double[] expectedSlopes = new double[] { 0.637, 0.637, 0.619, 0.614, 0.629 };
-        double[] expectedIntercepts = new double[] { 3.091, 3.091, 2.993, 2.905, 2.603 }; // check the numbers closer later
+        double[] expectedIntercepts = new double[] { 3.091, 3.091, 2.993, 2.905, 2.603 };
         long memSizeCapInBytes = (long) 100.0 * RBMSizeEstimator.BYTES_IN_MB;
         double delta = 0.01;
         for (int i = 0; i < mods.length; i++) {
@@ -295,4 +295,106 @@ public void testMemoryCapValueInitialization() {
         }
 
     }
+
+    public void testRemovalLogic() throws Exception {
+        RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX;
+        int modulo = moduloValue.getValue();
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L);
+
+        // Test standard sequence: add K1, K2, K3 which all transform to C, then:
+        // Remove K3
+        // Remove K2, re-add it, re-remove it twice (duplicate should do nothing)
+        // Remove K1, which should finally actually remove everything
+        int c = -42;
+        int k1 = c + modulo;
+        int k2 = c + 2 * modulo;
+        int k3 = c + 3 * modulo;
+        kls.add(k1);
+        assertTrue(kls.contains(k1));
+        assertTrue(kls.contains(k3));
+        kls.add(k2);
+        CounterMetric numCollisions = kls.getNumCollisionsForValue(k2);
+        assertNotNull(numCollisions);
+        assertEquals(2, numCollisions.count());
+        kls.add(k3);
+        assertEquals(3, numCollisions.count());
+        assertEquals(1, kls.getSize());
+
+        boolean removed = kls.remove(k3);
+        assertFalse(removed);
+        HashSet<Integer> removalSet = kls.getRemovalSetForValue(k3);
+        assertEquals(1, removalSet.size());
+        assertTrue(removalSet.contains(k3));
+        assertEquals(2, numCollisions.count());
+        assertEquals(1, kls.getSize());
+
+        removed = kls.remove(k2);
+        assertFalse(removed);
+        assertEquals(2, removalSet.size());
+        assertTrue(removalSet.contains(k2));
+        assertEquals(1, numCollisions.count());
+        assertEquals(1, kls.getSize());
+
+        kls.add(k2);
+        assertEquals(1, removalSet.size());
+        assertFalse(removalSet.contains(k2));
+        assertEquals(2, numCollisions.count());
+        assertEquals(1, kls.getSize());
+
+        removed = kls.remove(k2);
+        assertFalse(removed);
+        assertEquals(2, removalSet.size());
+        assertTrue(removalSet.contains(k2));
+        assertEquals(1, numCollisions.count());
+        assertEquals(1, kls.getSize());
+
+        removed = kls.remove(k2);
+        assertFalse(removed);
+        assertEquals(2, removalSet.size());
+        assertTrue(removalSet.contains(k2));
+        assertEquals(1, numCollisions.count());
+        assertEquals(1, kls.getSize());
+
+        removed = kls.remove(k1);
+        assertTrue(removed);
+        assertNull(kls.getRemovalSetForValue(k1));
+        assertNull(kls.getNumCollisionsForValue(k1));
+        assertEquals(0, kls.getSize());
+    }
+
+    public void testRemovalLogicWithHashCollision() throws Exception {
+        RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX;
+        int modulo = moduloValue.getValue();
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L);
+
+        // Test adding K1 twice (maybe two keys hash to K1), then removing it twice.
+        // We expect it to be unable to remove the last one, but there should be no false negatives.
+        int c = 77;
+        int k1 = c + modulo;
+        int k2 = c + 2 * modulo;
+        kls.add(k1);
+        kls.add(k2);
+        CounterMetric numCollisions = kls.getNumCollisionsForValue(k1);
+        assertEquals(2, numCollisions.count());
+        kls.add(k1);
+        assertEquals(3, numCollisions.count());
+
+        boolean removed = kls.remove(k1);
+        assertFalse(removed);
+        HashSet<Integer> removalSet = kls.getRemovalSetForValue(k1);
+        assertTrue(removalSet.contains(k1));
+        assertEquals(2, numCollisions.count());
+
+        removed = kls.remove(k2);
+        assertFalse(removed);
+        assertTrue(removalSet.contains(k2));
+        assertEquals(1, numCollisions.count());
+
+        removed = kls.remove(k1);
+        assertFalse(removed);
+        assertTrue(removalSet.contains(k1));
+        assertEquals(1, numCollisions.count());
+        assertTrue(kls.contains(k1));
+        assertTrue(kls.contains(k2)); 
+    }
 }

From 8e51749863b7a808c2a9b396725c661a041f7d4f Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 6 Nov 2023 15:17:55 -0800
Subject: [PATCH 045/120] Integrated keystore

---
 .../cache/tier/EhCacheDiskCachingTier.java      | 17 +++++++++++++++--
 .../cache/tier/keystore/KeyLookupStore.java     | 10 +++++-----
 .../tier/keystore/RBMIntKeyLookupStore.java     | 11 +++++------
 3 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index e7e52c7fbccbf..845eb4cc4f051 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -12,6 +12,7 @@
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
@@ -39,6 +40,7 @@
 import org.ehcache.event.EventType;
 import org.ehcache.expiry.ExpiryPolicy;
 import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
+import org.opensearch.core.common.unit.ByteSizeValue;
 
 public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
 
@@ -85,6 +87,7 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
     // Defines how many segments the disk cache is separated into. Higher number achieves greater concurrency but
     // will hold that many file pointers.
     public final Setting<Integer> DISK_SEGMENTS;
+    private final RBMIntKeyLookupStore keystore;
 
     private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.keyType = Objects.requireNonNull(builder.keyType, "Key type shouldn't be null");
@@ -108,6 +111,11 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.DISK_SEGMENTS = Setting.intSetting(builder.settingPrefix + ".ehcache.disk.segments", 16, 1, 32);
         this.cacheManager = buildCacheManager();
         this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
+
+        // IndicesRequestCache gets 1%, of which we allocate 5% to the keystore = 0.05%
+        // TODO: how do we change this automatically based on INDICES_CACHE_QUERY_SIZE setting?
+        Setting<ByteSizeValue> keystoreSizeSetting = Setting.memorySizeSetting(builder.settingPrefix + ".tiered.disk.keystore_size", "0.05%");
+        this.keystore = new RBMIntKeyLookupStore(keystoreSizeSetting.get(this.settings).getBytes());
     }
 
     private PersistentCacheManager buildCacheManager() {
@@ -176,13 +184,16 @@ private CacheEventListenerConfigurationBuilder getListenerConfiguration(Builder<
 
     @Override
     public V get(K key) {
-        // Optimize it by adding key store.
-        return cache.get(key);
+        if (keystore.contains(key.hashCode())) { // Check in-memory store of key hashes to avoid unnecessary disk seek
+            return cache.get(key);
+        }
+        return null;
     }
 
     @Override
     public void put(K key, V value) {
         cache.put(key, value);
+        keystore.add(key.hashCode());
     }
 
     @Override
@@ -195,6 +206,7 @@ public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception
     public void invalidate(K key) {
         // There seems to be an thread leak issue while calling this and then closing cache.
         cache.remove(key);
+        keystore.remove(key.hashCode());
     }
 
     @Override
@@ -211,6 +223,7 @@ public void setRemovalListener(RemovalListener<K, V> removalListener) {
     @Override
     public void invalidateAll() {
         // Clear up files.
+        keystore.clear();
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
index 552698e0293eb..a08174b5b46d3 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
@@ -47,7 +47,7 @@ public interface KeyLookupStore<T> {
      * @return true if the value was added, false if it wasn't added because of a
      * collision or if it was already present.
      */
-    boolean add(T value) throws Exception;
+    boolean add(T value);
 
     /**
      * Checks if the transformation of the value is in the keystore.
@@ -55,7 +55,7 @@ public interface KeyLookupStore<T> {
      * @return true if the value was found, false otherwise. Due to collisions, false positives are
      * possible, but there should be no false negatives unless forceRemove() is called.
      */
-    boolean contains(T value) throws Exception;
+    boolean contains(T value);
 
     /**
      * Returns the transformed version of the input value, that would be used to stored it in the keystore.
@@ -72,7 +72,7 @@ public interface KeyLookupStore<T> {
      * @param value The value to attempt to remove.
      * @return true if the value was removed, false if it wasn't.
      */
-    boolean remove(T value) throws Exception;
+    boolean remove(T value);
 
     /**
      * Returns the number of distinct values stored in the internal data structure.
@@ -123,10 +123,10 @@ public interface KeyLookupStore<T> {
      * Also resets all stats related to adding.
      * @param newValues The keys that should be in the reset structure.
      */
-    void regenerateStore(T[] newValues) throws Exception;
+    void regenerateStore(T[] newValues);
 
     /**
      * Deletes all keys and resets all stats related to adding.
      */
-    void clear() throws Exception;
+    void clear();
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index f9696e62a88b7..9dba9e8165d40 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -120,7 +120,7 @@ private void handleCollisions(int transformedValue) {
     }
 
     @Override
-    public boolean add(Integer value) throws Exception {
+    public boolean add(Integer value) {
         if (value == null) {
             return false;
         }
@@ -159,7 +159,7 @@ public boolean add(Integer value) throws Exception {
     }
 
     @Override
-    public boolean contains(Integer value) throws Exception {
+    public boolean contains(Integer value) {
         if (value == null) {
             return false;
         }
@@ -185,10 +185,9 @@ public Integer getInternalRepresentation(Integer value) {
      * may cause undefined behavior, including future false negatives!!
      * @param value The value to attempt to remove.
      * @return true if the value was removed, false otherwise
-     * @throws Exception
      */
     @Override
-    public boolean remove(Integer value) throws Exception {
+    public boolean remove(Integer value) {
         if (value == null) {
             return false;
         }
@@ -290,7 +289,7 @@ public boolean isFull() {
     }
 
     @Override
-    public void regenerateStore(Integer[] newValues) throws Exception {
+    public void regenerateStore(Integer[] newValues) {
         rbm.clear();
         collidedIntCounters = new HashMap<>();
         removalSets = new HashMap<>();
@@ -308,7 +307,7 @@ public void regenerateStore(Integer[] newValues) throws Exception {
     }
 
     @Override
-    public void clear() throws Exception {
+    public void clear() {
         regenerateStore(new Integer[] {});
     }
 

From 50999681945885101f2965665deda50b7581be62 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 6 Nov 2023 15:20:25 -0800
Subject: [PATCH 046/120] removed exceptions from interface signatures

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../common/cache/tier/keystore/KeyLookupStore.java    | 10 +++++-----
 .../cache/tier/keystore/RBMIntKeyLookupStore.java     | 11 +++++------
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
index 552698e0293eb..a08174b5b46d3 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
@@ -47,7 +47,7 @@ public interface KeyLookupStore<T> {
      * @return true if the value was added, false if it wasn't added because of a
      * collision or if it was already present.
      */
-    boolean add(T value) throws Exception;
+    boolean add(T value);
 
     /**
      * Checks if the transformation of the value is in the keystore.
@@ -55,7 +55,7 @@ public interface KeyLookupStore<T> {
      * @return true if the value was found, false otherwise. Due to collisions, false positives are
      * possible, but there should be no false negatives unless forceRemove() is called.
      */
-    boolean contains(T value) throws Exception;
+    boolean contains(T value);
 
     /**
      * Returns the transformed version of the input value, that would be used to stored it in the keystore.
@@ -72,7 +72,7 @@ public interface KeyLookupStore<T> {
      * @param value The value to attempt to remove.
      * @return true if the value was removed, false if it wasn't.
      */
-    boolean remove(T value) throws Exception;
+    boolean remove(T value);
 
     /**
      * Returns the number of distinct values stored in the internal data structure.
@@ -123,10 +123,10 @@ public interface KeyLookupStore<T> {
      * Also resets all stats related to adding.
      * @param newValues The keys that should be in the reset structure.
      */
-    void regenerateStore(T[] newValues) throws Exception;
+    void regenerateStore(T[] newValues);
 
     /**
      * Deletes all keys and resets all stats related to adding.
      */
-    void clear() throws Exception;
+    void clear();
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index f9696e62a88b7..9dba9e8165d40 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -120,7 +120,7 @@ private void handleCollisions(int transformedValue) {
     }
 
     @Override
-    public boolean add(Integer value) throws Exception {
+    public boolean add(Integer value) {
         if (value == null) {
             return false;
         }
@@ -159,7 +159,7 @@ public boolean add(Integer value) throws Exception {
     }
 
     @Override
-    public boolean contains(Integer value) throws Exception {
+    public boolean contains(Integer value) {
         if (value == null) {
             return false;
         }
@@ -185,10 +185,9 @@ public Integer getInternalRepresentation(Integer value) {
      * may cause undefined behavior, including future false negatives!!
      * @param value The value to attempt to remove.
      * @return true if the value was removed, false otherwise
-     * @throws Exception
      */
     @Override
-    public boolean remove(Integer value) throws Exception {
+    public boolean remove(Integer value) {
         if (value == null) {
             return false;
         }
@@ -290,7 +289,7 @@ public boolean isFull() {
     }
 
     @Override
-    public void regenerateStore(Integer[] newValues) throws Exception {
+    public void regenerateStore(Integer[] newValues) {
         rbm.clear();
         collidedIntCounters = new HashMap<>();
         removalSets = new HashMap<>();
@@ -308,7 +307,7 @@ public void regenerateStore(Integer[] newValues) throws Exception {
     }
 
     @Override
-    public void clear() throws Exception {
+    public void clear() {
         regenerateStore(new Integer[] {});
     }
 

From 21dab6f6421b7d91959b0858375ea399d1601fbf Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 4 Oct 2023 13:29:03 -0700
Subject: [PATCH 047/120] Adds caching policy

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .idea/runConfigurations/Debug_OpenSearch.xml  |   6 +-
 .../common/cache/tier/CacheTierPolicy.java    |  56 ++++++++
 .../cache/tier/DiskTierTookTimePolicy.java    |  91 ++++++++++++
 .../common/settings/ClusterSettings.java      |   5 +-
 .../opensearch/indices/CacheTierPolicy.java   |   0
 .../opensearch/indices/CheckDataResult.java   |   0
 .../opensearch/indices/DummyIRCPolicy.java    |   0
 .../IndicesRequestCacheDiskTierPolicy.java    |   0
 .../IndicesRequestCacheTookTimePolicy.java    |   0
 ...ndicesRequestCacheDiskTierPolicyTests.java | 132 ++++++++++++++++++
 10 files changed, 284 insertions(+), 6 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/CacheTierPolicy.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/CheckDataResult.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/DummyIRCPolicy.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/IndicesRequestCacheDiskTierPolicy.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/IndicesRequestCacheTookTimePolicy.java
 create mode 100644 server/src/test/java/org/opensearch/common/cache/tier/IndicesRequestCacheDiskTierPolicyTests.java

diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml
index c18046f873477..0d8bf59823acf 100644
--- a/.idea/runConfigurations/Debug_OpenSearch.xml
+++ b/.idea/runConfigurations/Debug_OpenSearch.xml
@@ -6,10 +6,6 @@
     <option name="HOST" value="localhost" />
     <option name="PORT" value="5005" />
     <option name="AUTO_RESTART" value="true" />
-    <RunnerSettings RunnerId="Debug">
-      <option name="DEBUG_PORT" value="5005" />
-      <option name="LOCAL" value="false" />
-    </RunnerSettings>
     <method v="2" />
   </configuration>
-</component>
\ No newline at end of file
+</component>
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
new file mode 100644
index 0000000000000..f242eeb6c53e0
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
@@ -0,0 +1,56 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.core.common.bytes.BytesReference;
+
+import java.io.IOException;
+
+public interface CacheTierPolicy<T> {
+    /**
+     * Determines whether this policy allows the data into its cache tier, based on the contents of the BytesReference
+     * which can be deserialized into class T.
+     * @param data A BytesReference which can be deserialized into class T
+     * @return A CheckDataResult object containing whether the data is admitted, and if it isn't, the reason.
+     * @throws IOException if the input can't be deserialized to the right class.
+     */
+    boolean checkData(BytesReference data) throws IOException;
+
+    /**
+     * Convert the BytesReference into the type T that is used to check entry into the cache.
+     * @param data The BytesReference
+     * @return The BytesReference converted to type T
+     * @throws IOException if the input can't be deserialized to the right class.
+     */
+    T convertFromBytesReference(BytesReference data) throws IOException;
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
new file mode 100644
index 0000000000000..de0730e4c4098
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
@@ -0,0 +1,91 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.search.query.QuerySearchResult;
+
+import java.io.IOException;
+
+/**
+ * A cache tier policy which accepts queries whose took time is greater than some threshold,
+ * which is specified as a dynamic cluster-level setting. The threshold should be set to approximately
+ * the time it takes to get a result from the cache tier.
+ */
+public class DiskTierTookTimePolicy implements CacheTierPolicy<QuerySearchResult> {
+    public static final Setting<TimeValue> INDICES_REQUEST_CACHE_DISK_TOOKTIME_THRESHOLD_SETTING = Setting.positiveTimeSetting(
+        "index.requests.cache.disk.tooktime.threshold",
+        new TimeValue(10),
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    private TimeValue threshold;
+
+    public DiskTierTookTimePolicy(Settings settings, ClusterSettings clusterSettings) {
+        this.threshold = INDICES_REQUEST_CACHE_DISK_TOOKTIME_THRESHOLD_SETTING.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(INDICES_REQUEST_CACHE_DISK_TOOKTIME_THRESHOLD_SETTING, this::setThreshold);
+    }
+
+    protected void setThreshold(TimeValue threshold) { // public so that we can manually set value in unit test
+        this.threshold = threshold;
+    }
+
+    @Override
+    public QuerySearchResult convertFromBytesReference(BytesReference data) throws IOException {
+        try {
+            return new QuerySearchResult(data.streamInput());
+        } catch (IllegalStateException ise) {
+            throw new IOException(ise);
+        }
+    }
+
+    @Override
+    public boolean checkData(BytesReference data) throws IOException {
+        QuerySearchResult qsr = convertFromBytesReference(data);
+        Long tookTimeNanos = qsr.getTookTimeNanos();
+        if (tookTimeNanos == null) {
+            return true;
+            // Received a null took time -> this QSR is from an old version which does not have took time, we should accept it
+        }
+        TimeValue tookTime = TimeValue.timeValueNanos(qsr.getTookTimeNanos());
+        if (tookTime.compareTo(threshold) < 0) { // negative -> tookTime is shorter than threshold
+            return false;
+        }
+        return true;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 74224d66400da..9690c25cecf6e 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -112,6 +112,7 @@
 import org.opensearch.indices.IndexingMemoryController;
 import org.opensearch.indices.IndicesQueryCache;
 import org.opensearch.indices.IndicesRequestCache;
+import org.opensearch.indices.IndicesRequestCacheTookTimePolicy;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.ShardLimitValidator;
 import org.opensearch.indices.analysis.HunspellService;
@@ -671,7 +672,9 @@ public void apply(Settings value, Settings current, Settings previous) {
                 // Remote cluster state settings
                 RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING,
                 RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING,
-                IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING
+                IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
+
+                IndicesRequestCacheTookTimePolicy.INDICES_REQUEST_CACHE_DISK_TOOKTIME_THRESHOLD_SETTING
             )
         )
     );
diff --git a/server/src/main/java/org/opensearch/indices/CacheTierPolicy.java b/server/src/main/java/org/opensearch/indices/CacheTierPolicy.java
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/server/src/main/java/org/opensearch/indices/CheckDataResult.java b/server/src/main/java/org/opensearch/indices/CheckDataResult.java
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/server/src/main/java/org/opensearch/indices/DummyIRCPolicy.java b/server/src/main/java/org/opensearch/indices/DummyIRCPolicy.java
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCacheDiskTierPolicy.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCacheDiskTierPolicy.java
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCacheTookTimePolicy.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCacheTookTimePolicy.java
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/IndicesRequestCacheDiskTierPolicyTests.java b/server/src/test/java/org/opensearch/common/cache/tier/IndicesRequestCacheDiskTierPolicyTests.java
new file mode 100644
index 0000000000000..0c1df13692079
--- /dev/null
+++ b/server/src/test/java/org/opensearch/common/cache/tier/IndicesRequestCacheDiskTierPolicyTests.java
@@ -0,0 +1,132 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.indices;
+
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TotalHits;
+import org.opensearch.action.OriginalIndices;
+import org.opensearch.action.OriginalIndicesTests;
+import org.opensearch.action.search.SearchRequest;
+import org.opensearch.common.UUIDs;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.Strings;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.search.DocValueFormat;
+import org.opensearch.search.SearchShardTarget;
+import org.opensearch.search.internal.AliasFilter;
+import org.opensearch.search.internal.ShardSearchContextId;
+import org.opensearch.search.internal.ShardSearchRequest;
+import org.opensearch.search.query.QuerySearchResult;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+
+public class IndicesRequestCacheDiskTierPolicyTests extends OpenSearchTestCase {
+    private DiskTierTookTimePolicy getTookTimePolicy() {
+        // dummy settings
+        Settings dummySettings = Settings.EMPTY;
+        ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        return new DiskTierTookTimePolicy(dummySettings, dummyClusterSettings);
+    }
+
+    public void testQSRSetupFunction() throws IOException {
+        long ttn = 100000000000L;
+        BytesReference qsrBytes = getQSRBytesReference(ttn);
+        QuerySearchResult qsr = new QuerySearchResult(qsrBytes.streamInput());
+        assertEquals(ttn, qsr.getTookTimeNanos());
+    }
+
+    public void testBadBytesReference() throws Exception {
+        BytesReference badQSR = new BytesArray("I love bytes!!!");
+        DiskTierTookTimePolicy tookTimePolicy = getTookTimePolicy();
+        assertThrows(IOException.class, () -> tookTimePolicy.checkData(badQSR));
+    }
+
+    public void testTookTimePolicy() throws Exception {
+        DiskTierTookTimePolicy tookTimePolicy = getTookTimePolicy();
+
+        // manually set threshold for test
+        double threshMillis = 10;
+        long shortMillis = (long) (0.9 * threshMillis);
+        long longMillis = (long) (1.5 * threshMillis);
+        tookTimePolicy.setThreshold(new TimeValue((long) threshMillis));
+        BytesReference shortQSR = getQSRBytesReference(shortMillis * 1000000);
+        BytesReference longQSR = getQSRBytesReference(longMillis * 1000000);
+
+        boolean shortResult = tookTimePolicy.checkData(shortQSR);
+        assertFalse(shortResult);
+        boolean longResult = tookTimePolicy.checkData(longQSR);
+        assertTrue(longResult);
+    }
+
+    private BytesReference getQSRBytesReference(long tookTimeNanos) throws IOException {
+        // setup from QuerySearchResultTests.java
+        ShardId shardId = new ShardId("index", "uuid", randomInt());
+        SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean());
+        ShardSearchRequest shardSearchRequest = new ShardSearchRequest(
+            OriginalIndicesTests.randomOriginalIndices(),
+            searchRequest,
+            shardId,
+            1,
+            new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1.0f,
+            randomNonNegativeLong(),
+            null,
+            new String[0]
+        );
+        ShardSearchContextId id = new ShardSearchContextId(UUIDs.base64UUID(), randomLong());
+        QuerySearchResult result = new QuerySearchResult(
+            id,
+            new SearchShardTarget("node", shardId, null, OriginalIndices.NONE),
+            shardSearchRequest
+        );
+        TopDocs topDocs = new TopDocs(new TotalHits(randomLongBetween(0, Long.MAX_VALUE), TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]);
+        result.topDocs(new TopDocsAndMaxScore(topDocs, randomBoolean() ? Float.NaN : randomFloat()), new DocValueFormat[0]);
+
+        result.setTookTimeNanos(tookTimeNanos);
+        BytesStreamOutput out = new BytesStreamOutput();
+        // it appears to need a boolean and then a ShardSearchContextId written to the stream before the QSR in order to deserialize?
+        out.writeBoolean(false);
+        id.writeTo(out);
+
+        result.writeToNoId(out);
+        return out.bytes();
+    }
+}

From f149eec5a03c05deeb05eb609d0d05f911715d3c Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 6 Nov 2023 16:06:32 -0800
Subject: [PATCH 048/120] Added first-draft integration for caching policy
 which will be changed when serializer is merged to make it generic

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../cache/tier/EhCacheDiskCachingTier.java    | 38 ++++++++++++++++++-
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index 845eb4cc4f051..c06e31d77be7e 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -19,8 +19,11 @@
 import org.opensearch.common.unit.TimeValue;
 
 import java.io.File;
+import java.io.IOException;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Objects;
 import java.util.Optional;
@@ -40,7 +43,9 @@
 import org.ehcache.event.EventType;
 import org.ehcache.expiry.ExpiryPolicy;
 import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
+import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.search.query.QuerySearchResult;
 
 public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
 
@@ -88,6 +93,8 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
     // will hold that many file pointers.
     public final Setting<Integer> DISK_SEGMENTS;
     private final RBMIntKeyLookupStore keystore;
+    private final List<CacheTierPolicy<QuerySearchResult>> policies;
+    // TODO: Generics? Might just remove <T> from CacheTierPolicy, but we do want to enforce it otherwise there will be errors
 
     private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.keyType = Objects.requireNonNull(builder.keyType, "Key type shouldn't be null");
@@ -116,6 +123,8 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         // TODO: how do we change this automatically based on INDICES_CACHE_QUERY_SIZE setting?
         Setting<ByteSizeValue> keystoreSizeSetting = Setting.memorySizeSetting(builder.settingPrefix + ".tiered.disk.keystore_size", "0.05%");
         this.keystore = new RBMIntKeyLookupStore(keystoreSizeSetting.get(this.settings).getBytes());
+
+        this.policies = Objects.requireNonNull(builder.policies);
     }
 
     private PersistentCacheManager buildCacheManager() {
@@ -192,8 +201,14 @@ public V get(K key) {
 
     @Override
     public void put(K key, V value) {
-        cache.put(key, value);
-        keystore.add(key.hashCode());
+        try {
+            if (checkPolicies(value)) {
+                cache.put(key, value);
+                keystore.add(key.hashCode());
+            }
+        } catch (IOException e) {
+            // Ignore for now, figure out what to do here
+        }
     }
 
     @Override
@@ -252,6 +267,19 @@ public void close() {
         }
     }
 
+    private boolean checkPolicies(V value) throws IOException {
+        for (CacheTierPolicy<QuerySearchResult> policy : policies) {
+            if (!policy.checkData((BytesReference) value)) {
+                // TODO: Change caching policy to take generic input
+                // (This will most likely have to happen after serializer is merged, by using the byte[] the value serializer
+                // produces to create a stream which we can pass to the caching policy, thereby avoiding having to make two diff streams
+                // in checking caching policy and also putting into the disk tier)
+                return false;
+            }
+        }
+        return true;
+    }
+
     /**
      * Wrapper over Ehcache original listener to listen to desired events and notify desired subscribers.
      * @param <K> Type of key
@@ -355,6 +383,7 @@ public static class Builder<K, V> {
 
         // Provides capability to make ehCache event listener to run in sync mode. Used for testing too.
         private boolean isEventListenerModeSync;
+        private final ArrayList<CacheTierPolicy<QuerySearchResult>> policies = new ArrayList<>();
 
         public Builder() {}
 
@@ -412,6 +441,11 @@ public EhCacheDiskCachingTier.Builder<K, V> setIsEventListenerModeSync(boolean i
             return this;
         }
 
+        public EhCacheDiskCachingTier.Builder<K, V> withPolicy(CacheTierPolicy<QuerySearchResult> policy) {
+            policies.add(policy);
+            return this;
+        }
+
         public EhCacheDiskCachingTier<K, V> build() {
             return new EhCacheDiskCachingTier<>(this);
         }

From ee7301a686f9ab880229e29ce8b93a7ac25d5f57 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 6 Nov 2023 17:37:53 -0800
Subject: [PATCH 049/120] Added unit test for ehcache tier's took time policy
 integration

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../core/common/bytes/BytesReference.java     |  4 +-
 .../tier/DiskTierTookTimePolicyTests.java     |  3 +-
 .../tier/EhCacheDiskCachingTierTests.java     | 37 +++++++++++++++++++
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
index 9d24d3653397b..5b53e1d9f8bae 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
@@ -45,6 +45,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 
@@ -54,7 +55,8 @@
  * @opensearch.api
  */
 @PublicApi(since = "1.0.0")
-public interface BytesReference extends Comparable<BytesReference>, ToXContentFragment {
+public interface BytesReference extends Comparable<BytesReference>, ToXContentFragment, Serializable {
+    // TODO: Remove "Serializable" once we merge in the serializer PR!
 
     /**
      * Convert an {@link XContentBuilder} into a BytesReference. This method closes the builder,
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java b/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
index 3810bb699ca84..7133599a7800d 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
@@ -97,7 +97,8 @@ public void testTookTimePolicy() throws Exception {
         assertTrue(longResult);
     }
 
-    private BytesReference getQSRBytesReference(long tookTimeNanos) throws IOException {
+     static BytesReference getQSRBytesReference(long tookTimeNanos) throws IOException {
+        // package-private, also used by EhCacheDiskCachingTierTests.java
         // setup from QuerySearchResultTests.java
         ShardId shardId = new ShardId("index", "uuid", randomInt());
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean());
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
index 804f236264daa..ba5c99469f231 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
@@ -9,8 +9,10 @@
 package org.opensearch.common.cache.tier;
 
 import org.opensearch.common.cache.RemovalListener;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.env.NodeEnvironment;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 
@@ -220,6 +222,41 @@ public boolean isLoaded() {
         }
     }
 
+    public void testThresholdPolicy() throws Exception {
+        long slowTookTimeNanos = 10000000000L; // 10 seconds
+        BytesReference slowResult = DiskTierTookTimePolicyTests.getQSRBytesReference(slowTookTimeNanos);
+
+        long fastTookTimeNanos = 100000L; // 100 microseconds
+        BytesReference fastResult = DiskTierTookTimePolicyTests.getQSRBytesReference(fastTookTimeNanos);
+
+        long thresholdMillis = 10;
+
+        // For this unit test, set the policy's threshold directly rather than from cluster settings
+        ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        DiskTierTookTimePolicy policy = new DiskTierTookTimePolicy(Settings.EMPTY, dummyClusterSettings);
+        policy.setThreshold(new TimeValue(thresholdMillis));
+
+        Settings settings = Settings.builder().build();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            EhCacheDiskCachingTier<String, BytesReference> tier = new EhCacheDiskCachingTier.Builder<String, BytesReference>()
+                .setKeyType(String.class)
+                .setValueType(BytesReference.class)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setSettings(settings)
+                .setThreadPoolAlias("ehcacheTest")
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setSettingPrefix(SETTING_PREFIX)
+                .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
+                .withPolicy(policy)
+                .build();
+            tier.put("slow", slowResult);
+            assertEquals(slowResult, tier.get("slow")); // key "slow" is found because the policy accepted it
+            tier.put("fast", fastResult);
+            assertNull(tier.get("fast")); // key "fast" -> null because the policy rejected it
+            tier.close();
+        }
+    }
+
     private RemovalListener<String, String> removalListener(AtomicInteger counter) {
         return notification -> counter.incrementAndGet();
     }

From ee11ed805a17ea3ddadda7e19c6596b1a020624e Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 8 Nov 2023 12:09:06 -0800
Subject: [PATCH 050/120] Reworked caching policy to sit at tiered cache
 service level and use V -> W generic transformation function to inspect cache
 values

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../common/cache/tier/CacheTierPolicy.java    |  27 +--
 .../cache/tier/DiskTierTookTimePolicy.java    |  31 ++--
 .../cache/tier/EhCacheDiskCachingTier.java    |  36 +---
 .../common/cache/tier/TieredCacheService.java |  12 +-
 .../TieredCacheSpilloverStrategyService.java  |  70 ++++++--
 .../indices/IndicesRequestCache.java          |  32 +++-
 .../opensearch/indices/IndicesService.java    |   2 +-
 .../tier/DiskTierTookTimePolicyTests.java     |  26 +--
 .../tier/EhCacheDiskCachingTierTests.java     |   4 +-
 ...redCacheSpilloverStrategyServiceTests.java | 166 ++++++++++++++++--
 .../indices/IndicesRequestCacheTests.java     |  79 ++++++++-
 11 files changed, 370 insertions(+), 115 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
index afe3b13525b7d..8af9326d4a033 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
@@ -38,19 +38,24 @@
 
 public interface CacheTierPolicy<T> {
     /**
-     * Determines whether this policy allows the data into its cache tier, based on the contents of the BytesReference
-     * which can be deserialized into class T.
-     * @param data A BytesReference which can be deserialized into class T
-     * @return A CheckDataResult object containing whether the data is admitted, and if it isn't, the reason.
-     * @throws IOException if the input can't be deserialized to the right class.
+     * Determines whether this policy allows the data into its cache tier.
+     * @param data The data to check
+     * @return true if accepted, otherwise false
      */
-    boolean checkData(BytesReference data) throws IOException;
+    boolean checkData(T data);
 
     /**
-     * Convert the BytesReference into the type T that is used to check entry into the cache.
-     * @param data The BytesReference
-     * @return The BytesReference converted to type T
-     * @throws IOException if the input can't be deserialized to the right class.
+     * Checks whether the policy is currently being enforced.
      */
-    T convertFromBytesReference(BytesReference data) throws IOException;
+    boolean isActive();
+
+    /**
+     * Starts enforcing the policy if it is not already being enforced.
+     */
+    void activate();
+
+    /**
+     * Stops enforcing the policy. The policy will allow all input.
+     */
+    void deactivate();
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
index 7d3aa3e53ba93..417a40f002e6a 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
@@ -55,10 +55,12 @@ public class DiskTierTookTimePolicy implements CacheTierPolicy<QuerySearchResult
     );
 
     private TimeValue threshold;
+    boolean isActive;
 
     public DiskTierTookTimePolicy(Settings settings, ClusterSettings clusterSettings) {
         this.threshold = DISK_TOOKTIME_THRESHOLD_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(DISK_TOOKTIME_THRESHOLD_SETTING, this::setThreshold);
+        isActive = true;
     }
 
     protected void setThreshold(TimeValue threshold) { // public so that we can manually set value in unit test
@@ -66,17 +68,10 @@ protected void setThreshold(TimeValue threshold) { // public so that we can manu
     }
 
     @Override
-    public QuerySearchResult convertFromBytesReference(BytesReference data) throws IOException {
-        try {
-            return new QuerySearchResult(data.streamInput());
-        } catch (IllegalStateException ise) {
-            throw new IOException(ise);
+    public boolean checkData(QuerySearchResult qsr) {
+        if (!isActive) {
+            return true;
         }
-    }
-
-    @Override
-    public boolean checkData(BytesReference data) throws IOException {
-        QuerySearchResult qsr = convertFromBytesReference(data);
         Long tookTimeNanos = qsr.getTookTimeNanos();
         if (tookTimeNanos == null) {
             return true;
@@ -88,4 +83,20 @@ public boolean checkData(BytesReference data) throws IOException {
         }
         return true;
     }
+
+    @Override
+    public boolean isActive() {
+        return isActive;
+    }
+
+    @Override
+    public void activate() {
+        isActive = true;
+
+    }
+
+    @Override
+    public void deactivate() {
+        isActive = false;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index c06e31d77be7e..295590d7094c2 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -93,8 +93,6 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
     // will hold that many file pointers.
     public final Setting<Integer> DISK_SEGMENTS;
     private final RBMIntKeyLookupStore keystore;
-    private final List<CacheTierPolicy<QuerySearchResult>> policies;
-    // TODO: Generics? Might just remove <T> from CacheTierPolicy, but we do want to enforce it otherwise there will be errors
 
     private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.keyType = Objects.requireNonNull(builder.keyType, "Key type shouldn't be null");
@@ -123,8 +121,6 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         // TODO: how do we change this automatically based on INDICES_CACHE_QUERY_SIZE setting?
         Setting<ByteSizeValue> keystoreSizeSetting = Setting.memorySizeSetting(builder.settingPrefix + ".tiered.disk.keystore_size", "0.05%");
         this.keystore = new RBMIntKeyLookupStore(keystoreSizeSetting.get(this.settings).getBytes());
-
-        this.policies = Objects.requireNonNull(builder.policies);
     }
 
     private PersistentCacheManager buildCacheManager() {
@@ -193,7 +189,7 @@ private CacheEventListenerConfigurationBuilder getListenerConfiguration(Builder<
 
     @Override
     public V get(K key) {
-        if (keystore.contains(key.hashCode())) { // Check in-memory store of key hashes to avoid unnecessary disk seek
+        if (keystore.contains(key.hashCode()) || keystore.isFull()) { // Check in-memory store of key hashes to avoid unnecessary disk seek
             return cache.get(key);
         }
         return null;
@@ -201,14 +197,8 @@ public V get(K key) {
 
     @Override
     public void put(K key, V value) {
-        try {
-            if (checkPolicies(value)) {
-                cache.put(key, value);
-                keystore.add(key.hashCode());
-            }
-        } catch (IOException e) {
-            // Ignore for now, figure out what to do here
-        }
+            cache.put(key, value);
+            keystore.add(key.hashCode());
     }
 
     @Override
@@ -267,19 +257,6 @@ public void close() {
         }
     }
 
-    private boolean checkPolicies(V value) throws IOException {
-        for (CacheTierPolicy<QuerySearchResult> policy : policies) {
-            if (!policy.checkData((BytesReference) value)) {
-                // TODO: Change caching policy to take generic input
-                // (This will most likely have to happen after serializer is merged, by using the byte[] the value serializer
-                // produces to create a stream which we can pass to the caching policy, thereby avoiding having to make two diff streams
-                // in checking caching policy and also putting into the disk tier)
-                return false;
-            }
-        }
-        return true;
-    }
-
     /**
      * Wrapper over Ehcache original listener to listen to desired events and notify desired subscribers.
      * @param <K> Type of key
@@ -383,8 +360,6 @@ public static class Builder<K, V> {
 
         // Provides capability to make ehCache event listener to run in sync mode. Used for testing too.
         private boolean isEventListenerModeSync;
-        private final ArrayList<CacheTierPolicy<QuerySearchResult>> policies = new ArrayList<>();
-
         public Builder() {}
 
         public EhCacheDiskCachingTier.Builder<K, V> setMaximumWeightInBytes(long sizeInBytes) {
@@ -441,11 +416,6 @@ public EhCacheDiskCachingTier.Builder<K, V> setIsEventListenerModeSync(boolean i
             return this;
         }
 
-        public EhCacheDiskCachingTier.Builder<K, V> withPolicy(CacheTierPolicy<QuerySearchResult> policy) {
-            policies.add(policy);
-            return this;
-        }
-
         public EhCacheDiskCachingTier<K, V> build() {
             return new EhCacheDiskCachingTier<>(this);
         }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
index 31d58510206f0..24fb9774fcb05 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
@@ -15,8 +15,11 @@
  * flavors like spillover etc.
  * @param <K> Type of key
  * @param <V> Type of value
+ * @param <W> Type that V can be transformed into to inspect its contents to apply policies.
+ *           For example, if V is BytesReference, W might be QuerySearchResult.
+ *           Can be the same as V if no transformation is necessary
  */
-public interface TieredCacheService<K, V> {
+public interface TieredCacheService<K, V, W> {
 
     V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception;
 
@@ -31,4 +34,11 @@ public interface TieredCacheService<K, V> {
     OnHeapCachingTier<K, V> getOnHeapCachingTier();
 
     Optional<DiskCachingTier<K, V>> getDiskCachingTier();
+
+    /**
+     * Function transforming values of type V to type W so they can be inspected for contents before applying disk-tier policies.
+     * @param value
+     * @return
+     */
+    W preDiskCachingPolicyFunction(V value);
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 153dbf9b330f5..7279c14ca3213 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -8,10 +8,15 @@
 
 package org.opensearch.common.cache.tier;
 
+import org.opensearch.common.cache.Cache;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.indices.IndicesRequestCache;
+import org.opensearch.search.query.QuerySearchResult;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
@@ -23,8 +28,9 @@
  * cache items to disk tier cache.
  * @param <K> Type of key
  * @param <V> Type of value
+ * @param <W> Type that V can be unpacked into for inspection by policies. Can be the same as V.
  */
-public class TieredCacheSpilloverStrategyService<K, V> implements TieredCacheService<K, V>, RemovalListener<K, V> {
+public class TieredCacheSpilloverStrategyService<K, V, W> implements TieredCacheService<K, V, W>, RemovalListener<K, V> {
 
     private final OnHeapCachingTier<K, V> onHeapCachingTier;
 
@@ -38,13 +44,17 @@ public class TieredCacheSpilloverStrategyService<K, V> implements TieredCacheSer
      * Maintains caching tiers in order of get calls.
      */
     private final List<CachingTier<K, V>> cachingTierList;
+    private final List<CacheTierPolicy<W>> policies;
+    private final Function<V, W> transformationFunction;
 
-    private TieredCacheSpilloverStrategyService(Builder<K, V> builder) {
+    private TieredCacheSpilloverStrategyService(Builder<K, V, W> builder) {
         this.onHeapCachingTier = Objects.requireNonNull(builder.onHeapCachingTier);
         this.diskCachingTier = Optional.ofNullable(builder.diskCachingTier);
         this.tieredCacheEventListener = Objects.requireNonNull(builder.tieredCacheEventListener);
         this.cachingTierList = this.diskCachingTier.map(diskTier -> Arrays.asList(onHeapCachingTier, diskTier))
             .orElse(List.of(onHeapCachingTier));
+        this.policies = Objects.requireNonNull(builder.policies);
+        this.transformationFunction = Objects.requireNonNull(builder.transformationFunction);
         setRemovalListeners();
     }
 
@@ -130,10 +140,12 @@ public void onRemoval(RemovalNotification<K, V> notification) {
         if (RemovalReason.EVICTED.equals(notification.getRemovalReason())) {
             switch (notification.getTierType()) {
                 case ON_HEAP:
-                    diskCachingTier.ifPresent(diskTier -> {
-                        diskTier.put(notification.getKey(), notification.getValue());
-                        tieredCacheEventListener.onCached(notification.getKey(), notification.getValue(), TierType.DISK);
-                    });
+                    if (checkPolicies(notification.getValue())) {
+                        diskCachingTier.ifPresent(diskTier -> {
+                            diskTier.put(notification.getKey(), notification.getValue());
+                            tieredCacheEventListener.onCached(notification.getKey(), notification.getValue(), TierType.DISK);
+                        });
+                    }
                     break;
                 default:
                     break;
@@ -152,6 +164,21 @@ public Optional<DiskCachingTier<K, V>> getDiskCachingTier() {
         return this.diskCachingTier;
     }
 
+    @Override
+    public W preDiskCachingPolicyFunction(V value) {
+        return transformationFunction.apply(value);
+    }
+
+    boolean checkPolicies(V value) {
+        W unpacked = preDiskCachingPolicyFunction(value);
+        for (CacheTierPolicy<W> policy : policies) {
+            if (!policy.checkData(unpacked)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     /**
      * Register this service as a listener to removal events from different caching tiers.
      */
@@ -197,31 +224,50 @@ public static class CacheValue<V> {
      * Builder object
      * @param <K> Type of key
      * @param <V> Type of value
+     * @param <W> Type that V can be unpacked into for inspection by policies. Can be the same as V.
      */
-    public static class Builder<K, V> {
+    public static class Builder<K, V, W> {
         private OnHeapCachingTier<K, V> onHeapCachingTier;
         private DiskCachingTier<K, V> diskCachingTier;
         private TieredCacheEventListener<K, V> tieredCacheEventListener;
+        private ArrayList<CacheTierPolicy<W>> policies = new ArrayList<>();
+        private Function<V, W> transformationFunction;
 
         public Builder() {}
 
-        public Builder<K, V> setOnHeapCachingTier(OnHeapCachingTier<K, V> onHeapCachingTier) {
+        public Builder<K, V, W> setOnHeapCachingTier(OnHeapCachingTier<K, V> onHeapCachingTier) {
             this.onHeapCachingTier = onHeapCachingTier;
             return this;
         }
 
-        public Builder<K, V> setOnDiskCachingTier(DiskCachingTier<K, V> diskCachingTier) {
+        public Builder<K, V, W> setOnDiskCachingTier(DiskCachingTier<K, V> diskCachingTier) {
             this.diskCachingTier = diskCachingTier;
             return this;
         }
 
-        public Builder<K, V> setTieredCacheEventListener(TieredCacheEventListener<K, V> tieredCacheEventListener) {
+        public Builder<K, V, W> setTieredCacheEventListener(TieredCacheEventListener<K, V> tieredCacheEventListener) {
             this.tieredCacheEventListener = tieredCacheEventListener;
             return this;
         }
 
-        public TieredCacheSpilloverStrategyService<K, V> build() {
-            return new TieredCacheSpilloverStrategyService<K, V>(this);
+        public Builder<K, V, W> withPolicy(CacheTierPolicy<W> policy) {
+            this.policies.add(policy);
+            return this;
+        }
+
+        // Add multiple policies at once
+        public Builder<K, V, W> withPolicies(List<CacheTierPolicy<W>> policiesList) {
+            this.policies.addAll(policiesList);
+            return this;
+        }
+
+        public Builder<K, V, W> withPreDiskCachingPolicyFunction(Function<V, W> function) {
+            this.transformationFunction = function;
+            return this;
+        }
+
+        public TieredCacheSpilloverStrategyService<K, V, W> build() {
+            return new TieredCacheSpilloverStrategyService<K, V, W>(this);
         }
     }
 
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 2f06394ab7120..881f6a2f8bb9c 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -40,6 +40,7 @@
 import org.apache.lucene.util.RamUsageEstimator;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
 import org.opensearch.common.cache.tier.OnHeapCachingTier;
 import org.opensearch.common.cache.tier.OpenSearchOnHeapCache;
 import org.opensearch.common.cache.tier.TierType;
@@ -48,6 +49,7 @@
 import org.opensearch.common.cache.tier.TieredCacheService;
 import org.opensearch.common.cache.tier.TieredCacheSpilloverStrategyService;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
@@ -58,6 +60,7 @@
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.search.query.QuerySearchResult;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -68,6 +71,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
+import java.util.function.Function;
 
 /**
  * The indices request cache allows to cache a shard level request stage responses, helping with improving
@@ -113,11 +117,11 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
     private final Set<CleanupKey> keysToClean = ConcurrentCollections.newConcurrentSet();
     private final ByteSizeValue size;
     private final TimeValue expire;
-    private final TieredCacheService<Key, BytesReference> tieredCacheService;
+    private final TieredCacheService<Key, BytesReference, QuerySearchResult> tieredCacheService;
 
     private final IndicesService indicesService;
 
-    IndicesRequestCache(Settings settings, IndicesService indicesService) {
+    IndicesRequestCache(Settings settings, IndicesService indicesService, ClusterSettings clusterSettings) {
         this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
         this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
         long sizeInBytes = size.getBytes();
@@ -127,10 +131,21 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             (k, v) -> k.ramBytesUsed() + v.ramBytesUsed()
         ).setMaximumWeight(sizeInBytes).setExpireAfterAccess(expire).build();
 
+        Function<BytesReference, QuerySearchResult> transformationFunction = (data) -> {
+            try {
+                return convertBytesReferenceToQSR(data);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        };
+
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
-        tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>().setOnHeapCachingTier(
+        tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference, QuerySearchResult>().setOnHeapCachingTier(
             openSearchOnHeapCache
-        ).setTieredCacheEventListener(this).build();
+        ).setTieredCacheEventListener(this)
+            .withPreDiskCachingPolicyFunction(transformationFunction)
+            .withPolicy(new DiskTierTookTimePolicy(settings, clusterSettings))
+            .build();
         this.indicesService = indicesService;
     }
 
@@ -209,6 +224,15 @@ void invalidate(CacheEntity cacheEntity, DirectoryReader reader, BytesReference
         tieredCacheService.invalidate(new Key(cacheEntity, cacheKey, readerCacheKeyId));
     }
 
+    // Passed to TieredCacheService as its V -> W transformation function for inspecting BytesReferences in policies
+    public static QuerySearchResult convertBytesReferenceToQSR(BytesReference data) throws IOException {
+        try {
+            return new QuerySearchResult(data.streamInput());
+        } catch (IllegalStateException ise) {
+            throw new IOException(ise);
+        }
+    }
+
     /**
      * Loader for the request cache
      *
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index 34e0153dbe730..16891ae5e8bb2 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -405,7 +405,7 @@ public IndicesService(
         this.shardsClosedTimeout = settings.getAsTime(INDICES_SHARDS_CLOSED_TIMEOUT, new TimeValue(1, TimeUnit.DAYS));
         this.analysisRegistry = analysisRegistry;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
-        this.indicesRequestCache = new IndicesRequestCache(settings, this);
+        this.indicesRequestCache = new IndicesRequestCache(settings, this, clusterService.getClusterSettings());
         this.indicesQueryCache = new IndicesQueryCache(settings);
         this.mapperRegistry = mapperRegistry;
         this.namedWriteableRegistry = namedWriteableRegistry;
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java b/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
index 7133599a7800d..b55aaf28cfd4a 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
@@ -69,17 +69,9 @@ private DiskTierTookTimePolicy getTookTimePolicy() {
 
     public void testQSRSetupFunction() throws IOException {
         Long ttn = 100000000000L;
-        BytesReference qsrBytes = getQSRBytesReference(ttn);
-        QuerySearchResult qsr = new QuerySearchResult(qsrBytes.streamInput());
+        QuerySearchResult qsr = getQSR(ttn);
         assertEquals(ttn, qsr.getTookTimeNanos());
     }
-
-    public void testBadBytesReference() throws Exception {
-        BytesReference badQSR = new BytesArray("I love bytes!!!");
-        DiskTierTookTimePolicy tookTimePolicy = getTookTimePolicy();
-        assertThrows(IOException.class, () -> tookTimePolicy.checkData(badQSR));
-    }
-
     public void testTookTimePolicy() throws Exception {
         DiskTierTookTimePolicy tookTimePolicy = getTookTimePolicy();
 
@@ -88,8 +80,8 @@ public void testTookTimePolicy() throws Exception {
         long shortMillis = (long) (0.9 * threshMillis);
         long longMillis = (long) (1.5 * threshMillis);
         tookTimePolicy.setThreshold(new TimeValue((long) threshMillis));
-        BytesReference shortQSR = getQSRBytesReference(shortMillis * 1000000);
-        BytesReference longQSR = getQSRBytesReference(longMillis * 1000000);
+        QuerySearchResult shortQSR = getQSR(shortMillis * 1000000);
+        QuerySearchResult longQSR = getQSR(longMillis * 1000000);
 
         boolean shortResult = tookTimePolicy.checkData(shortQSR);
         assertFalse(shortResult);
@@ -97,8 +89,8 @@ public void testTookTimePolicy() throws Exception {
         assertTrue(longResult);
     }
 
-     static BytesReference getQSRBytesReference(long tookTimeNanos) throws IOException {
-        // package-private, also used by EhCacheDiskCachingTierTests.java
+     public static QuerySearchResult getQSR(long tookTimeNanos) {
+        // package-private, also used by IndicesRequestCacheTests.java
         // setup from QuerySearchResultTests.java
         ShardId shardId = new ShardId("index", "uuid", randomInt());
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean());
@@ -123,12 +115,6 @@ static BytesReference getQSRBytesReference(long tookTimeNanos) throws IOExceptio
         result.topDocs(new TopDocsAndMaxScore(topDocs, randomBoolean() ? Float.NaN : randomFloat()), new DocValueFormat[0]);
 
         result.setTookTimeNanos(tookTimeNanos);
-        BytesStreamOutput out = new BytesStreamOutput();
-        // it appears to need a boolean and then a ShardSearchContextId written to the stream before the QSR in order to deserialize?
-        out.writeBoolean(false);
-        id.writeTo(out);
-
-        result.writeToNoId(out);
-        return out.bytes();
+        return result;
     }
 }
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
index ba5c99469f231..471383bfa4c71 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
@@ -222,7 +222,7 @@ public boolean isLoaded() {
         }
     }
 
-    public void testThresholdPolicy() throws Exception {
+    /*public void testThresholdPolicy() throws Exception {
         long slowTookTimeNanos = 10000000000L; // 10 seconds
         BytesReference slowResult = DiskTierTookTimePolicyTests.getQSRBytesReference(slowTookTimeNanos);
 
@@ -255,7 +255,7 @@ public void testThresholdPolicy() throws Exception {
             assertNull(tier.get("fast")); // key "fast" -> null because the policy rejected it
             tier.close();
         }
-    }
+    }*/
 
     private RemovalListener<String, String> removalListener(AtomicInteger counter) {
         return notification -> counter.incrementAndGet();
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java b/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
index bb7a22cc26037..d775025e9c04c 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
@@ -15,21 +15,27 @@
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.EnumMap;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
 
 public class TieredCacheSpilloverStrategyServiceTests extends OpenSearchTestCase {
 
     public void testComputeAndAbsentWithoutAnyOnHeapCacheEviction() throws Exception {
         int onHeapCacheSize = randomIntBetween(10, 30);
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
-        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = intializeTieredCacheService(
+        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = intializeTieredCacheService(
             onHeapCacheSize,
             randomIntBetween(1, 4),
-            eventListener
+            eventListener,
+                null,
+            null
         );
         int numOfItems1 = randomIntBetween(1, onHeapCacheSize / 2 - 1);
         List<String> keys = new ArrayList<>();
@@ -70,10 +76,12 @@ public void testComputeAndAbsentWithEvictionsFromOnHeapCache() throws Exception
         int diskCacheSize = randomIntBetween(60, 100);
         int totalSize = onHeapCacheSize + diskCacheSize;
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
-        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = intializeTieredCacheService(
+        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = intializeTieredCacheService(
             onHeapCacheSize,
             diskCacheSize,
-            eventListener
+            eventListener,
+            null,
+            null
         );
 
         // Put values in cache more than it's size and cause evictions from onHeap.
@@ -137,10 +145,12 @@ public void testComputeAndAbsentWithEvictionsFromBothTier() throws Exception {
         int totalSize = onHeapCacheSize + diskCacheSize;
 
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
-        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = intializeTieredCacheService(
+        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = intializeTieredCacheService(
             onHeapCacheSize,
             diskCacheSize,
-            eventListener
+            eventListener,
+            null,
+            null
         );
 
         int numOfItems = randomIntBetween(totalSize + 1, totalSize * 3);
@@ -158,10 +168,12 @@ public void testGetAndCount() throws Exception {
         int totalSize = onHeapCacheSize + diskCacheSize;
 
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
-        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = intializeTieredCacheService(
+        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = intializeTieredCacheService(
             onHeapCacheSize,
             diskCacheSize,
-            eventListener
+            eventListener,
+            null,
+            null
         );
 
         int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize);
@@ -198,9 +210,14 @@ public void testGetAndCount() throws Exception {
     public void testWithDiskTierNull() throws Exception {
         int onHeapCacheSize = randomIntBetween(10, 30);
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
-        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = new TieredCacheSpilloverStrategyService.Builder<
+        Function<String, String> identityFunction = (String value) -> { return value; };
+        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = new TieredCacheSpilloverStrategyService.Builder<
             String,
-            String>().setOnHeapCachingTier(new MockOnHeapCacheTier<>(onHeapCacheSize)).setTieredCacheEventListener(eventListener).build();
+            String,
+            String>().setOnHeapCachingTier(new MockOnHeapCacheTier<>(onHeapCacheSize))
+            .setTieredCacheEventListener(eventListener)
+            .withPreDiskCachingPolicyFunction(identityFunction)
+            .build();
         int numOfItems = randomIntBetween(onHeapCacheSize + 1, onHeapCacheSize * 3);
         for (int iter = 0; iter < numOfItems; iter++) {
             TieredCacheLoader<String, String> tieredCacheLoader = getTieredCacheLoader();
@@ -212,6 +229,91 @@ public void testWithDiskTierNull() throws Exception {
         assertEquals(0, eventListener.enumMap.get(TierType.DISK).missCount.count());
     }
 
+    public void testTransformationFunctionAndDiskTierPolicy() throws Exception {
+        // For transformation function, return the string reversed
+        // For policy function, allow if what it receives starts with "a" (so original string ends with "a") and string is even length
+        Function<String, String> reverseStringFunction = (String value) -> {
+            return new StringBuilder(value).reverse().toString();
+        };
+        ArrayList<CacheTierPolicy<String>> policies = new ArrayList<>();
+        policies.add(new AllowFirstLetterA());
+        policies.add(new AllowEvenLengths());
+
+        int onHeapCacheSize = 0;
+        int diskCacheSize = 10000;
+        MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
+        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = intializeTieredCacheService(
+            onHeapCacheSize,
+            diskCacheSize,
+            eventListener,
+            reverseStringFunction,
+            policies
+        );
+
+        Map<String, String> keyValuePairs = new HashMap<>();
+        Map<String, Boolean> expectedOutputs = new HashMap<>();
+        keyValuePairs.put("key1", "dcba");
+        expectedOutputs.put("key1", true);
+        keyValuePairs.put("key2", "edcba");
+        expectedOutputs.put("key2", false);
+        keyValuePairs.put("key3", "abc");
+        expectedOutputs.put("key3", false);
+        keyValuePairs.put("key4", "ab");
+        expectedOutputs.put("key4", false);
+        keyValuePairs.put("key5", "");
+        expectedOutputs.put("key5", false);
+
+        TieredCacheLoader<String, String> loader = getTieredCacheLoaderWithKeyValueMap(keyValuePairs);
+
+        for (String key : keyValuePairs.keySet()) {
+            Boolean expectedOutput = expectedOutputs.get(key);
+            String value = spilloverStrategyService.computeIfAbsent(key, loader);
+            assertEquals(keyValuePairs.get(key), value);
+            String result = spilloverStrategyService.get(key);
+            if (expectedOutput) {
+                // Should retrieve from disk tier if it was accepted
+                assertEquals(keyValuePairs.get(key), result);
+            } else {
+                // Should miss as heap tier size = 0 and the policy rejected it
+                assertNull(result);
+            }
+        }
+    }
+
+    private static class AllowFirstLetterA implements CacheTierPolicy<String> {
+        @Override
+        public boolean checkData(String data) {
+            try {
+                return (data.charAt(0) == 'a');
+            } catch (StringIndexOutOfBoundsException e) {
+                return false;
+            }
+        }
+        @Override
+        public boolean isActive() {
+            return true;
+        }
+        @Override
+        public void activate() {}
+        @Override
+        public void deactivate() {}
+    }
+
+    private static class AllowEvenLengths implements CacheTierPolicy<String> {
+        @Override
+        public boolean checkData(String data) {
+            return data.length() % 2 == 0;
+        }
+        @Override
+        public boolean isActive() {
+            return true;
+        }
+        @Override
+        public void activate() {}
+        @Override
+        public void deactivate() {}
+    }
+
     private TieredCacheLoader<String, String> getTieredCacheLoader() {
         return new TieredCacheLoader<String, String>() {
             boolean isLoaded = false;
@@ -229,16 +331,50 @@ public boolean isLoaded() {
         };
     }
 
-    private TieredCacheSpilloverStrategyService<String, String> intializeTieredCacheService(
+    private TieredCacheLoader<String, String> getTieredCacheLoaderWithKeyValueMap(Map<String, String> map) {
+        return new TieredCacheLoader<String, String>() {
+            boolean isLoaded;
+            @Override
+            public String load(String key) throws Exception {
+                isLoaded = true;
+                return map.get(key);
+            }
+
+            @Override
+            public boolean isLoaded() {
+                return isLoaded;
+            }
+        };
+    }
+
+    private TieredCacheSpilloverStrategyService<String, String, String> intializeTieredCacheService(
         int onHeapCacheSize,
-        int diksCacheSize,
-        TieredCacheEventListener<String, String> cacheEventListener
+        int diskCacheSize,
+        TieredCacheEventListener<String, String> cacheEventListener,
+        Function<String, String> transformationFunction, // If passed null, default to identity function
+        List<CacheTierPolicy<String>> policies // If passed null, default to no policies (empty list)
     ) {
-        DiskCachingTier<String, String> diskCache = new MockDiskCachingTier<>(diksCacheSize);
+        DiskCachingTier<String, String> diskCache = new MockDiskCachingTier<>(diskCacheSize);
         OnHeapCachingTier<String, String> openSearchOnHeapCache = new MockOnHeapCacheTier<>(onHeapCacheSize);
-        return new TieredCacheSpilloverStrategyService.Builder<String, String>().setOnHeapCachingTier(openSearchOnHeapCache)
+
+        Function<String, String> function;
+        if (transformationFunction != null) {
+            function = transformationFunction;
+        } else {
+            function = (String value) -> value;
+        }
+
+        List<CacheTierPolicy<String>> policiesToUse = new ArrayList<>();
+        if (policies != null) {
+            policiesToUse = policies;
+        }
+
+
+        return new TieredCacheSpilloverStrategyService.Builder<String, String, String>().setOnHeapCachingTier(openSearchOnHeapCache)
             .setOnDiskCachingTier(diskCache)
             .setTieredCacheEventListener(cacheEventListener)
+            .withPreDiskCachingPolicyFunction(function)
+            .withPolicies(policiesToUse)
             .build();
     }
 
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 18ec013711f22..41be95c5b098f 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -41,16 +41,28 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.opensearch.action.OriginalIndices;
+import org.opensearch.action.OriginalIndicesTests;
+import org.opensearch.action.search.SearchRequest;
 import org.opensearch.common.CheckedSupplier;
+import org.opensearch.common.UUIDs;
+import org.opensearch.common.cache.tier.DiskTierTookTimePolicyTests;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
+import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.io.IOUtils;
+import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.AbstractBytesReference;
+import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -62,6 +74,12 @@
 import org.opensearch.index.cache.request.ShardRequestCache;
 import org.opensearch.index.query.TermQueryBuilder;
 import org.opensearch.index.shard.IndexShard;
+import org.opensearch.search.DocValueFormat;
+import org.opensearch.search.SearchShardTarget;
+import org.opensearch.search.internal.AliasFilter;
+import org.opensearch.search.internal.ShardSearchContextId;
+import org.opensearch.search.internal.ShardSearchRequest;
+import org.opensearch.search.query.QuerySearchResult;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 
 import java.io.IOException;
@@ -73,7 +91,8 @@ public class IndicesRequestCacheTests extends OpenSearchSingleNodeTestCase {
 
     public void testBasicOperationsCache() throws Exception {
         ShardRequestCache requestCacheStats = new ShardRequestCache();
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class));
+        ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
 
@@ -127,7 +146,8 @@ public void testBasicOperationsCache() throws Exception {
     }
 
     public void testCacheDifferentReaders() throws Exception {
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class));
+        ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
         AtomicBoolean indexShard = new AtomicBoolean(true);
         ShardRequestCache requestCacheStats = new ShardRequestCache();
         Directory dir = newDirectory();
@@ -222,8 +242,9 @@ public void testCacheDifferentReaders() throws Exception {
 
     public void testEviction() throws Exception {
         final ByteSizeValue size;
+        ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         {
-            IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class));
+            IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
             AtomicBoolean indexShard = new AtomicBoolean(true);
             ShardRequestCache requestCacheStats = new ShardRequestCache();
             Directory dir = newDirectory();
@@ -250,7 +271,8 @@ public void testEviction() throws Exception {
         }
         IndicesRequestCache cache = new IndicesRequestCache(
             Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), size.getBytes() + 1 + "b").build(),
-            null
+            null,
+            dummyClusterSettings
         );
         AtomicBoolean indexShard = new AtomicBoolean(true);
         ShardRequestCache requestCacheStats = new ShardRequestCache();
@@ -287,7 +309,8 @@ public void testEviction() throws Exception {
     }
 
     public void testClearAllEntityIdentity() throws Exception {
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class));
+        ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
         AtomicBoolean indexShard = new AtomicBoolean(true);
 
         ShardRequestCache requestCacheStats = new ShardRequestCache();
@@ -372,7 +395,8 @@ public BytesReference get() {
 
     public void testInvalidate() throws Exception {
         ShardRequestCache requestCacheStats = new ShardRequestCache();
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class));
+        ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
 
@@ -494,6 +518,49 @@ public void testSerializationDeserializationOfCacheKey() throws Exception {
 
     }
 
+    private static BytesReference getQSRBytesReference(long tookTimeNanos) throws IOException {
+        // unfortunately no good way to separate this out from DiskTierTookTimePolicyTests.getQSR() :(
+        ShardId shardId = new ShardId("index", "uuid", randomInt());
+        SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean());
+        ShardSearchRequest shardSearchRequest = new ShardSearchRequest(
+            OriginalIndicesTests.randomOriginalIndices(),
+            searchRequest,
+            shardId,
+            1,
+            new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1.0f,
+            randomNonNegativeLong(),
+            null,
+            new String[0]
+        );
+        ShardSearchContextId id = new ShardSearchContextId(UUIDs.base64UUID(), randomLong());
+        QuerySearchResult result = new QuerySearchResult(
+            id,
+            new SearchShardTarget("node", shardId, null, OriginalIndices.NONE),
+            shardSearchRequest
+        );
+        TopDocs topDocs = new TopDocs(new TotalHits(randomLongBetween(0, Long.MAX_VALUE), TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]);
+        result.topDocs(new TopDocsAndMaxScore(topDocs, randomBoolean() ? Float.NaN : randomFloat()), new DocValueFormat[0]);
+
+        result.setTookTimeNanos(tookTimeNanos);
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        // it appears to need a boolean and then a ShardSearchContextId written to the stream before the QSR in order to deserialize?
+        out.writeBoolean(false);
+        id.writeTo(out);
+        result.writeToNoId(out);
+        return out.bytes();
+    }
+
+    public void testBytesReferenceToQSRFunction() throws Exception {
+        BytesReference badQSR = new BytesArray("I love bytes!!!");
+        long ttn = 1000L;
+        BytesReference goodQSR = getQSRBytesReference(ttn);
+        assertThrows(IOException.class, () -> IndicesRequestCache.convertBytesReferenceToQSR(badQSR));
+        QuerySearchResult unpackedQSR = IndicesRequestCache.convertBytesReferenceToQSR(goodQSR);
+        assertEquals(ttn, (long) unpackedQSR.getTookTimeNanos());
+    }
+
     private class TestBytesReference extends AbstractBytesReference {
 
         int dummyValue;

From 5123142950e4e0e8346da1b00b3a23ef04abfba2 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 8 Nov 2023 14:23:30 -0800
Subject: [PATCH 051/120] Fixed issue with eviction test for IRC tests

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../cache/tier/TieredCacheSpilloverStrategyService.java  | 9 ++++++++-
 .../java/org/opensearch/indices/IndicesRequestCache.java | 8 ++------
 .../org/opensearch/indices/IndicesRequestCacheTests.java | 5 ++---
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 7279c14ca3213..5b0e0b7a98635 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -170,7 +170,14 @@ public W preDiskCachingPolicyFunction(V value) {
     }
 
     boolean checkPolicies(V value) {
-        W unpacked = preDiskCachingPolicyFunction(value);
+        W unpacked;
+        try {
+            unpacked = preDiskCachingPolicyFunction(value);
+        } catch (Exception e) {
+            // Bandaid solution: Accept any values that can't be unpacked into type W.
+            // TODO: Decide if this is reasonable, or if we should require the IRC to only accept BytesReferences which can be turned into QSRs.
+            return true;
+        }
         for (CacheTierPolicy<W> policy : policies) {
             if (!policy.checkData(unpacked)) {
                 return false;
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 881f6a2f8bb9c..58b15292b7c2c 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -225,12 +225,8 @@ void invalidate(CacheEntity cacheEntity, DirectoryReader reader, BytesReference
     }
 
     // Passed to TieredCacheService as its V -> W transformation function for inspecting BytesReferences in policies
-    public static QuerySearchResult convertBytesReferenceToQSR(BytesReference data) throws IOException {
-        try {
-            return new QuerySearchResult(data.streamInput());
-        } catch (IllegalStateException ise) {
-            throw new IOException(ise);
-        }
+    public static QuerySearchResult convertBytesReferenceToQSR(BytesReference data) throws IOException, IllegalStateException {
+        return new QuerySearchResult(data.streamInput());
     }
 
     /**
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 41be95c5b098f..300d6f9ccc172 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -41,7 +41,6 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
@@ -53,7 +52,6 @@
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.UUIDs;
-import org.opensearch.common.cache.tier.DiskTierTookTimePolicyTests;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
 import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
@@ -556,7 +554,8 @@ public void testBytesReferenceToQSRFunction() throws Exception {
         BytesReference badQSR = new BytesArray("I love bytes!!!");
         long ttn = 1000L;
         BytesReference goodQSR = getQSRBytesReference(ttn);
-        assertThrows(IOException.class, () -> IndicesRequestCache.convertBytesReferenceToQSR(badQSR));
+        //assertThrows(IOException.class, () -> IndicesRequestCache.convertBytesReferenceToQSR(badQSR));
+        // TODO: Uncomment this line if we decide to enforce IRC only accepting BytesReferences which can become QSRs.
         QuerySearchResult unpackedQSR = IndicesRequestCache.convertBytesReferenceToQSR(goodQSR);
         assertEquals(ttn, (long) unpackedQSR.getTookTimeNanos());
     }

From 4865827726cb1cce3783b9289c8ec69879bf4e5f Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 8 Nov 2023 15:11:55 -0800
Subject: [PATCH 052/120] first parts of cachevalue rework (nonfunctional)

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../common/cache/tier/CacheValue.java         | 30 ++++++++++++++++
 .../common/cache/tier/CachingTier.java        |  2 +-
 .../cache/tier/DiskTierRequestStats.java      | 35 +++++++++++++++++++
 .../cache/tier/OnHeapTierRequestStats.java    | 19 ++++++++++
 .../common/cache/tier/TierRequestStats.java   | 17 +++++++++
 .../TieredCacheSpilloverStrategyService.java  | 12 -------
 6 files changed, 102 insertions(+), 13 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/TierRequestStats.java

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java b/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
new file mode 100644
index 0000000000000..a7a7039cfc770
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+import org.opensearch.common.cache.tier.TierType;
+
+import java.util.Map;
+
+/**
+ * Represents a cache value, with its associated tier type where it is stored,
+ * and tier-specific stats for an individual request stored in a map.
+ * @param <V> Type of value.
+ */
+public class CacheValue<V> {
+    V value;
+    TierType source;
+    Map<String, Object> stats;
+
+    CacheValue(V value, TierType source, Map<String, Object> stats) {
+        this.value = value;
+        this.source = source;
+        this.stats = stats;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/CachingTier.java
index 48fd5deadc111..2e94cacd0f40a 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/CachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CachingTier.java
@@ -18,7 +18,7 @@
  */
 public interface CachingTier<K, V> {
 
-    V get(K key);
+    CacheValue<V> get(K key);
 
     void put(K key, V value);
 
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java
new file mode 100644
index 0000000000000..33f36202c3f84
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * A class created by disk tier implementations containing disk-specific stats for a single request.
+ */
+public class DiskTierRequestStats implements TierRequestStats {
+
+    private final long requestGetTimeNanos;
+    private final boolean requestReachedDisk;
+
+    public DiskTierRequestStats(long requestGetTimeNanos, boolean requestReachedDisk) {
+        this.requestReachedDisk = requestReachedDisk;
+        this.requestGetTimeNanos = requestGetTimeNanos;
+    }
+    @Override
+    public TierType getTierType() {
+        return TierType.DISK;
+    }
+
+    public long getRequestGetTimeNanos() {
+        return requestGetTimeNanos;
+    }
+
+    public boolean getRequestReachedDisk() {
+        return requestReachedDisk;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java b/server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java
new file mode 100644
index 0000000000000..8f37f9e8a3141
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java
@@ -0,0 +1,19 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * A class created by disk tier implementations containing disk-specific stats for a single request.
+ */
+public class OnHeapTierRequestStats implements TierRequestStats {
+    @Override
+    public TierType getTierType() {
+        return TierType.ON_HEAP;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TierRequestStats.java b/server/src/main/java/org/opensearch/common/cache/tier/TierRequestStats.java
new file mode 100644
index 0000000000000..d156be7b3f028
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TierRequestStats.java
@@ -0,0 +1,17 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * An interface for single-request tier-specific stats, which are produced on each request from a cache tier
+ * and then added to the correct shard's overall StatsHolder for the tier.
+ */
+public interface TierRequestStats {
+    TierType getTierType();
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 153dbf9b330f5..0fcf0f03dc0ed 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -179,19 +179,7 @@ private Function<K, CacheValue<V>> getValueFromTierCache(boolean trackStats) {
         };
     }
 
-    /**
-     * Represents a cache value along with its associated tier type where it is stored.
-     * @param <V> Type of value.
-     */
-    public static class CacheValue<V> {
-        V value;
-        TierType source;
 
-        CacheValue(V value, TierType source) {
-            this.value = value;
-            this.source = source;
-        }
-    }
 
     /**
      * Builder object

From 8fb86b7622bca74a2e4f5ab0da1070f2ba1e0df7 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 8 Nov 2023 16:29:03 -0800
Subject: [PATCH 053/120] Almost finished statsholder impl side, found and
 deleted a lot of old tiered stuff in indices package

---
 .../common/cache/tier/CacheValue.java         |  16 +-
 .../cache/tier/EhCacheDiskCachingTier.java    |  14 +-
 .../cache/tier/OpenSearchOnHeapCache.java     |   4 +-
 .../common/cache/tier/TierType.java           |  16 +-
 .../cache/tier/TieredCacheEventListener.java  |   5 +-
 .../TieredCacheSpilloverStrategyService.java  |  10 +-
 .../index/cache/request/DiskStatsHolder.java  |  99 ++++++
 .../cache/request/OnHeapStatsHolder.java      |  37 +++
 .../cache/request/RequestCacheStats.java      |   3 +-
 .../cache/request/ShardRequestCache.java      |   9 +-
 .../index/cache/request/StatsHolder.java      |  16 +-
 .../AbstractIndexShardCacheEntity.java        |   9 +-
 .../org/opensearch/indices/CachingTier.java   |  39 ---
 .../opensearch/indices/DiskCachingTier.java   |  22 --
 .../indices/DummyDiskCachingTier.java         |  58 ----
 .../indices/DummySerializableKey.java         |  54 ----
 .../indices/EhcacheDiskCachingTier.java       | 281 ------------------
 .../indices/EhcacheEventListener.java         |  70 -----
 .../org/opensearch/indices/EhcacheKey.java    |  50 ----
 .../indices/IndicesRequestCache.java          |  13 +-
 .../opensearch/indices/OnHeapCachingTier.java |  11 -
 .../indices/OpenSearchOnHeapCache.java        | 119 --------
 .../java/org/opensearch/indices/TierType.java |  26 --
 .../indices/TieredCacheEventListener.java     |  22 --
 .../indices/TieredCacheHandler.java           |  28 --
 .../opensearch/indices/TieredCacheLoader.java |  15 -
 .../TieredCacheSpilloverStrategyHandler.java  | 195 ------------
 ...redCacheSpilloverStrategyServiceTests.java |   8 +-
 .../cache/request/RequestCacheStatsTests.java |   1 -
 .../indices/IndicesServiceCloseTests.java     |   5 +-
 30 files changed, 226 insertions(+), 1029 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/index/cache/request/DiskStatsHolder.java
 create mode 100644 server/src/main/java/org/opensearch/index/cache/request/OnHeapStatsHolder.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/CachingTier.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/DiskCachingTier.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/DummyDiskCachingTier.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/DummySerializableKey.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/EhcacheKey.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/OnHeapCachingTier.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/OpenSearchOnHeapCache.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/TierType.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/TieredCacheEventListener.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/TieredCacheLoader.java
 delete mode 100644 server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java b/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
index a7a7039cfc770..baeac79c172d9 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
@@ -20,11 +20,23 @@
 public class CacheValue<V> {
     V value;
     TierType source;
-    Map<String, Object> stats;
+    TierRequestStats stats;
 
-    CacheValue(V value, TierType source, Map<String, Object> stats) {
+    CacheValue(V value, TierType source, TierRequestStats stats) {
         this.value = value;
         this.source = source;
         this.stats = stats;
     }
+
+    public V getValue() {
+        return value;
+    }
+
+    public TierType getSource() {
+        return source;
+    }
+
+    public TierRequestStats getStats() {
+        return stats;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index e7e52c7fbccbf..fd6b74a983fd8 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -175,9 +175,19 @@ private CacheEventListenerConfigurationBuilder getListenerConfiguration(Builder<
     }
 
     @Override
-    public V get(K key) {
+    public CacheValue<V> get(K key) {
         // Optimize it by adding key store.
-        return cache.get(key);
+        boolean reachedDisk = true; // TODO: Change this once we combine this with keystore integration
+        long now = System.nanoTime(); // Nanoseconds required; milliseconds might be too slow on an SSD
+
+        V value = cache.get(key);
+
+        long tookTime = -1L; // This value will be ignored by stats accumulator if reachedDisk is false anyway
+        if (reachedDisk) {
+            tookTime = System.nanoTime() - now;
+        }
+        DiskTierRequestStats stats = new DiskTierRequestStats(tookTime, reachedDisk);
+        return new CacheValue<>(value, TierType.DISK, stats);
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/OpenSearchOnHeapCache.java b/server/src/main/java/org/opensearch/common/cache/tier/OpenSearchOnHeapCache.java
index 22d2f769507cf..c7cd065674de7 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/OpenSearchOnHeapCache.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/OpenSearchOnHeapCache.java
@@ -66,8 +66,8 @@ public TierType getTierType() {
     }
 
     @Override
-    public V get(K key) {
-        return cache.get(key);
+    public CacheValue<V> get(K key) {
+        return new CacheValue<V>(cache.get(key), TierType.ON_HEAP, new OnHeapTierRequestStats());
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TierType.java b/server/src/main/java/org/opensearch/common/cache/tier/TierType.java
index ca61b636c1dda..e3c18f6a1b5a3 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TierType.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TierType.java
@@ -13,6 +13,18 @@
  */
 public enum TierType {
 
-    ON_HEAP,
-    DISK;
+    ON_HEAP("on_heap"),
+    DISK("disk");
+
+    private final String stringValue;
+
+    TierType(String stringValue) {
+        // Associate each TierType with a string representation, for use in API responses and elsewhere
+        this.stringValue = stringValue;
+    }
+
+    public String getStringValue() {
+        return this.stringValue;
+    }
 }
+
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheEventListener.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheEventListener.java
index 05b59bf16b282..d1c333e21818e 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheEventListener.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheEventListener.java
@@ -17,11 +17,12 @@
  */
 public interface TieredCacheEventListener<K, V> {
 
-    void onMiss(K key, TierType tierType);
+    void onMiss(K key, CacheValue<V> cacheValue);
 
     void onRemoval(RemovalNotification<K, V> notification);
 
-    void onHit(K key, V value, TierType tierType);
+    void onHit(K key, CacheValue<V> cacheValue);
 
     void onCached(K key, V value, TierType tierType);
+    // Since only get() produces a CacheValue with stats, no need to modify onCached or onRemoval to have the CacheValue.
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 0fcf0f03dc0ed..f8e037515ae6d 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -164,15 +164,15 @@ private void setRemovalListeners() {
     private Function<K, CacheValue<V>> getValueFromTierCache(boolean trackStats) {
         return key -> {
             for (CachingTier<K, V> cachingTier : cachingTierList) {
-                V value = cachingTier.get(key);
-                if (value != null) {
+                CacheValue<V> cacheValue = cachingTier.get(key);
+                if (cacheValue.value != null) {
                     if (trackStats) {
-                        tieredCacheEventListener.onHit(key, value, cachingTier.getTierType());
+                        tieredCacheEventListener.onHit(key, cacheValue);
                     }
-                    return new CacheValue<>(value, cachingTier.getTierType());
+                    return cacheValue; //new CacheValue<>(value, cachingTier.getTierType());
                 }
                 if (trackStats) {
-                    tieredCacheEventListener.onMiss(key, cachingTier.getTierType());
+                    tieredCacheEventListener.onMiss(key, cacheValue);
                 }
             }
             return null;
diff --git a/server/src/main/java/org/opensearch/index/cache/request/DiskStatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/DiskStatsHolder.java
new file mode 100644
index 0000000000000..094b358333908
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/cache/request/DiskStatsHolder.java
@@ -0,0 +1,99 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.cache.request;
+
+import org.opensearch.common.cache.tier.DiskTierRequestStats;
+import org.opensearch.common.cache.tier.TierRequestStats;
+import org.opensearch.common.cache.tier.TierType;
+import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+public class DiskStatsHolder extends StatsHolder {
+    final CounterMetric totalGetTime;
+    final CounterMetric totalDiskReaches; // Number of times a get() has actually reached the disk
+    public DiskStatsHolder() {
+        super();
+        this.totalGetTime = new CounterMetric();
+        this.totalDiskReaches = new CounterMetric();
+    }
+
+    public DiskStatsHolder(long memorySize, long evictions, long hitCount, long missCount, long entries, long totalGetTime, long totalDiskReaches) {
+        super(memorySize, evictions, hitCount, missCount, entries);
+        this.totalGetTime = new CounterMetric();
+        this.totalGetTime.inc(totalGetTime);
+        this.totalDiskReaches = new CounterMetric();
+        this.totalDiskReaches.inc(totalDiskReaches);
+    }
+
+    public DiskStatsHolder(StreamInput in) throws IOException {
+        this(
+            in.readVLong(),
+            in.readVLong(),
+            in.readVLong(),
+            in.readVLong(),
+            in.readVLong(),
+            in.readVLong(),
+            in.readVLong()
+        );
+    }
+
+    public long getTotalGetTime() {
+        return totalGetTime.count();
+    }
+
+    public long getTotalDiskReaches() {
+        return totalDiskReaches.count();
+    }
+
+    @Override
+    public void addRequestStats(TierRequestStats stats) {
+        assert stats.getTierType() == TierType.DISK;
+        DiskTierRequestStats diskStats = (DiskTierRequestStats) stats;
+        // TODO: Should this instead be accomplished with generics somehow
+        if (diskStats.getRequestReachedDisk()) {
+            this.totalDiskReaches.inc();
+            this.totalGetTime.inc(diskStats.getRequestGetTimeNanos());
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeVLong(totalGetTime.count());
+        out.writeVLong(totalDiskReaches.count());
+    }
+
+    @Override
+    public void add(StatsHolder stats) {
+        assert stats.getClass() == DiskStatsHolder.class;
+        super.add(stats);
+        DiskStatsHolder diskStats = (DiskStatsHolder) stats;
+        totalDiskReaches.inc(diskStats.totalDiskReaches.count());
+        totalGetTime.inc(diskStats.totalGetTime.count());
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        super.toXContent(builder, params);
+        builder.field(Fields.TOTAL_GET_TIME, getTotalGetTime());
+        builder.field(Fields.TOTAL_DISK_REACHES, getTotalDiskReaches());
+        return builder;
+    }
+
+    static final class Fields {
+        static final String TOTAL_GET_TIME = "total_get_time_nanos";
+        static final String TOTAL_DISK_REACHES = "total_disk_reaches";
+    }
+
+
+}
diff --git a/server/src/main/java/org/opensearch/index/cache/request/OnHeapStatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/OnHeapStatsHolder.java
new file mode 100644
index 0000000000000..5ef11f7850c5d
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/cache/request/OnHeapStatsHolder.java
@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.cache.request;
+
+import org.opensearch.common.cache.tier.TierRequestStats;
+import org.opensearch.core.common.io.stream.StreamInput;
+
+import java.io.IOException;
+
+
+/**
+ * This class accumulates stats for a single shard, for the on-heap tier.
+ * For now, on-heap tier has no unique stats, but future stats would be added here.
+ */
+public class OnHeapStatsHolder extends StatsHolder {
+    public OnHeapStatsHolder() {
+        super();
+    }
+
+    public OnHeapStatsHolder(long memorySize, long evictions, long hitCount, long missCount, long entries) {
+        super(memorySize, evictions, hitCount, missCount, entries);
+    }
+
+    public OnHeapStatsHolder(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public void addRequestStats(TierRequestStats stats) {}
+
+}
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index f37c7965a97c9..1907541621968 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -33,16 +33,15 @@
 package org.opensearch.index.cache.request;
 
 import org.opensearch.Version;
+import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.indices.TierType;
 
 import java.io.IOException;
-import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.Map;
 
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index dac5e818f85da..2a9ddf9c33618 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -33,6 +33,7 @@
 package org.opensearch.index.cache.request;
 
 import org.apache.lucene.util.Accountable;
+import org.opensearch.common.cache.tier.CacheValue;
 import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -58,12 +59,12 @@ public RequestCacheStats stats() {
         return new RequestCacheStats(statsHolder);
     }
 
-    public void onHit(TierType tierType) {
-        statsHolder.get(tierType).hitCount.inc();
+    public void onHit(CacheValue<BytesReference> cacheValue) {
+        statsHolder.get(cacheValue.getSource()).hitCount.inc();
     }
 
-    public void onMiss(TierType tierType) {
-        statsHolder.get(tierType).missCount.inc();
+    public void onMiss(CacheValue<BytesReference> cacheValue) {
+        statsHolder.get(cacheValue.getSource()).missCount.inc();
     }
 
     public void onCached(Accountable key, BytesReference value, TierType tierType) {
diff --git a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
index 92d057ab8fd9c..7596da018e842 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.index.cache.request;
 
+import org.opensearch.common.cache.tier.TierRequestStats;
+import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -19,13 +21,19 @@
 import java.io.IOException;
 import java.io.Serializable;
 
-public class StatsHolder implements Serializable, Writeable, ToXContentFragment {
+/**
+ * The basic StatsHolder class, which accumulates shard-level stats that are common across all tiers.
+ * Used in ShardRequestCache. Extending classes also handle tier-specific stats for each tier.
+ */
+public abstract class StatsHolder implements Serializable, Writeable, ToXContentFragment {
     final CounterMetric totalMetric;
     final CounterMetric evictionsMetric;
     final CounterMetric hitCount;
     final CounterMetric missCount;
     final CounterMetric entries;
 
+    public static final Map<TierType, >
+
     public StatsHolder() {
         this.totalMetric = new CounterMetric();
         this.evictionsMetric = new CounterMetric();
@@ -107,4 +115,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(RequestCacheStats.Fields.ENTRIES, getEntries());
         return builder;
     }
+
+    /**
+     * Used to add stats for a single request (get call) to existing stats
+     * @param stats Stats for a single request
+     */
+    public abstract void addRequestStats(TierRequestStats stats);
 }
diff --git a/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java b/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
index d9c256b4b4a94..ac12e6bdf389a 100644
--- a/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
+++ b/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.cache.tier.CacheValue;
 import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.index.cache.request.ShardRequestCache;
@@ -57,13 +58,13 @@ public final void onCached(IndicesRequestCache.Key key, BytesReference value, Ti
     }
 
     @Override
-    public final void onHit(TierType tierType) {
-        stats().onHit(tierType);
+    public final void onHit(CacheValue<BytesReference> cacheValue) {
+        stats().onHit(cacheValue);
     }
 
     @Override
-    public final void onMiss(TierType tierType) {
-        stats().onMiss(tierType);
+    public final void onMiss(CacheValue<BytesReference> cacheValue) {
+        stats().onMiss(cacheValue);
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/indices/CachingTier.java b/server/src/main/java/org/opensearch/indices/CachingTier.java
deleted file mode 100644
index 85596929cfd6b..0000000000000
--- a/server/src/main/java/org/opensearch/indices/CachingTier.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-import org.opensearch.common.cache.RemovalListener;
-
-/**
- * asdsadssa
- * @param <K>
- * @param <V>
- */
-public interface CachingTier<K, V> {
-
-    V get(K key);
-
-    void put(K key, V value);
-
-    V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception;
-
-    void invalidate(K key);
-
-    V compute(K key, TieredCacheLoader<K, V> loader) throws Exception;
-
-    void setRemovalListener(RemovalListener<K, V> removalListener);
-
-    void invalidateAll();
-
-    Iterable<K> keys();
-
-    int count();
-
-    TierType getTierType();
-}
diff --git a/server/src/main/java/org/opensearch/indices/DiskCachingTier.java b/server/src/main/java/org/opensearch/indices/DiskCachingTier.java
deleted file mode 100644
index b1b5b03ed13cd..0000000000000
--- a/server/src/main/java/org/opensearch/indices/DiskCachingTier.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-public interface DiskCachingTier<K, V> extends CachingTier<K, V> {
-    /**
-     * Closes the disk tier.
-     */
-    void close();
-
-    /**
-     * Get the EWMA time in milliseconds for a get().
-     * @return
-     */
-    double getTimeMillisEWMA();
-}
diff --git a/server/src/main/java/org/opensearch/indices/DummyDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/DummyDiskCachingTier.java
deleted file mode 100644
index 26a78b6c61920..0000000000000
--- a/server/src/main/java/org/opensearch/indices/DummyDiskCachingTier.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-import org.opensearch.common.cache.RemovalListener;
-
-import java.util.Collections;
-
-public class DummyDiskCachingTier<K, V> implements CachingTier<K, V> {
-
-    @Override
-    public V get(K key) {
-        return null;
-    }
-
-    @Override
-    public void put(K key, V value) {}
-
-    @Override
-    public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception {
-        return null;
-    }
-
-    @Override
-    public void invalidate(K key) {}
-
-    @Override
-    public V compute(K key, TieredCacheLoader<K, V> loader) throws Exception {
-        return null;
-    }
-
-    @Override
-    public void setRemovalListener(RemovalListener<K, V> removalListener) {}
-
-    @Override
-    public void invalidateAll() {}
-
-    @Override
-    public Iterable<K> keys() {
-        return Collections::emptyIterator;
-    }
-
-    @Override
-    public int count() {
-        return 0;
-    }
-
-    @Override
-    public TierType getTierType() {
-        return TierType.DISK;
-    }
-}
diff --git a/server/src/main/java/org/opensearch/indices/DummySerializableKey.java b/server/src/main/java/org/opensearch/indices/DummySerializableKey.java
deleted file mode 100644
index d69734f344d4e..0000000000000
--- a/server/src/main/java/org/opensearch/indices/DummySerializableKey.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-public class DummySerializableKey implements Serializable {
-    private Integer i;
-    private String s;
-
-    public DummySerializableKey(Integer i, String s) {
-        this.i = i;
-        this.s = s;
-    }
-
-    public int getI() {
-        return i;
-    }
-
-    public String getS() {
-        return s;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (o == this) {
-            return true;
-        }
-        if (!(o instanceof DummySerializableKey)) {
-            return false;
-        }
-        DummySerializableKey other = (DummySerializableKey) o;
-        return Objects.equals(this.i, other.i) && this.s.equals(other.s);
-    }
-
-    @Override
-    public final int hashCode() {
-        int result = 11;
-        if (i != null) {
-            result = 31 * result + i.hashCode();
-        }
-        if (s != null) {
-            result = 31 * result + s.hashCode();
-        }
-        return result;
-    }
-}
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java b/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
deleted file mode 100644
index 7d96f587bac65..0000000000000
--- a/server/src/main/java/org/opensearch/indices/EhcacheDiskCachingTier.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-import org.opensearch.common.ExponentiallyWeightedMovingAverage;
-import org.opensearch.common.cache.RemovalListener;
-import org.opensearch.common.cache.RemovalNotification;
-import org.opensearch.common.io.PathUtils;
-import org.opensearch.common.metrics.CounterMetric;
-import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.io.stream.BytesStreamInput;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-
-import org.ehcache.Cache;
-import org.ehcache.PersistentCacheManager;
-import org.ehcache.config.builders.CacheConfigurationBuilder;
-import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
-import org.ehcache.config.builders.CacheManagerBuilder;
-import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
-import org.ehcache.config.builders.ResourcePoolsBuilder;
-import org.ehcache.config.units.MemoryUnit;
-import org.ehcache.event.EventType;
-import org.ehcache.impl.config.executor.PooledExecutionServiceConfiguration;
-
-public class EhcacheDiskCachingTier
-    implements
-        DiskCachingTier<IndicesRequestCache.Key, BytesReference>,
-        RemovalListener<IndicesRequestCache.Key, BytesReference> {
-
-    public static HashMap<String, PersistentCacheManager> cacheManagers = new HashMap<>();
-    // Because of the way test cases are set up, each node may try to instantiate several disk caching tiers.
-    // Only one of them will be used, but there will be initialization errors when multiple cache managers try to
-    // use the same file path and create/get caches with the same alias. We resolve this with a static reference
-    // to a cache manager, which is populated if it is null and reused if it is not.
-    // (See https://stackoverflow.com/questions/53756412/ehcache-org-ehcache-statetransitionexception-persistence-directory-already-lo)
-    // To deal with IT cases, we need to create a manager per node, as otherwise nodes will try to reuse the same manager,
-    // so we get the correct cache manager by looking up the node ID in this map.
-    // I don't think any of this can happen in production, because nodes shouldn't share a JVM,
-    // and they should only instantiate their services once? But it's best to resolve it anyway.
-
-    private PersistentCacheManager cacheManager; // This is the manager this tier will actually use
-    private Cache<EhcacheKey, BytesReference> cache;
-    public final static String BASE_DISK_CACHE_FP = "disk_cache_tier";
-    // Placeholder. this should probably be defined somewhere else, since we need to change security.policy based on its value
-    // To accomodate test setups, where multiple nodes may exist on the same filesystem, we add the node ID to the end of this
-    // These will be subfolders of BASE_DISK_CACHE_FP
-    private final String diskCacheFP; // the one to use for this node
-    private RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener;
-    private ExponentiallyWeightedMovingAverage getTimeMillisEWMA;
-    private static final double GET_TIME_EWMA_ALPHA = 0.3; // This is the value used elsewhere in OpenSearch
-    private static final int MIN_WRITE_THREADS = 0;
-    private static final int MAX_WRITE_THREADS = 4; // Max number of threads for the PooledExecutionService which handles writes
-    private static final String cacheAlias = "diskTier";
-    private CounterMetric count; // number of entries in cache
-    private final EhcacheEventListener listener;
-    private final IndicesRequestCache indicesRequestCache; // only used to create new Keys
-    private final String nodeId;
-    // private RBMIntKeyLookupStore keystore;
-    // private CacheTierPolicy[] policies;
-    // private IndicesRequestCacheDiskTierPolicy policy;
-
-    public EhcacheDiskCachingTier(
-        long maxWeightInBytes,
-        long maxKeystoreWeightInBytes,
-        IndicesRequestCache indicesRequestCache,
-        String nodeId
-    ) {
-        this.count = new CounterMetric();
-        this.listener = new EhcacheEventListener(this, this);
-        this.indicesRequestCache = indicesRequestCache;
-        this.nodeId = nodeId;
-        this.diskCacheFP = PathUtils.get(BASE_DISK_CACHE_FP, nodeId).toString();
-        // I know this function warns it shouldn't often be used, we can fix it to use the roots once we pick a final FP
-
-        getManager();
-        try {
-            cacheManager.destroyCache(cacheAlias);
-        } catch (Exception e) {
-            System.out.println("Unable to destroy cache!!");
-            e.printStackTrace();
-            // do actual logging later
-        }
-        createCache(maxWeightInBytes);
-        this.getTimeMillisEWMA = new ExponentiallyWeightedMovingAverage(GET_TIME_EWMA_ALPHA, 10);
-
-        // this.keystore = new RBMIntKeyLookupStore((int) Math.pow(2, 28), maxKeystoreWeightInBytes);
-        // this.policies = new CacheTierPolicy[]{ new IndicesRequestCacheTookTimePolicy(settings, clusterSettings) };
-        // this.policy = new IndicesRequestCacheDiskTierPolicy(this.policies, true);
-    }
-
-    public void getManager() {
-        PersistentCacheManager oldCacheManager = cacheManagers.get(nodeId);
-        if (oldCacheManager != null) {
-            try {
-                try {
-                    oldCacheManager.close();
-                } catch (IllegalStateException e) {
-                    System.out.println("Cache was uninitialized, skipping close() and moving to destroy()");
-                }
-                oldCacheManager.destroy();
-            } catch (Exception e) {
-                System.out.println("Was unable to destroy existing cache manager");
-                e.printStackTrace();
-                // actual logging later
-            }
-        }
-        PooledExecutionServiceConfiguration threadConfig = PooledExecutionServiceConfigurationBuilder
-            .newPooledExecutionServiceConfigurationBuilder()
-            .defaultPool("default", MIN_WRITE_THREADS, MAX_WRITE_THREADS)
-            .build();
-
-        cacheManagers.put(
-            nodeId,
-            CacheManagerBuilder.newCacheManagerBuilder().using(threadConfig).with(CacheManagerBuilder.persistence(diskCacheFP)).build(true)
-        );
-        this.cacheManager = cacheManagers.get(nodeId);
-    }
-
-    private void createCache(long maxWeightInBytes) {
-        // our EhcacheEventListener should receive events every time an entry is changed
-        CacheEventListenerConfigurationBuilder listenerConfig = CacheEventListenerConfigurationBuilder.newEventListenerConfiguration(
-            listener,
-            EventType.EVICTED,
-            EventType.EXPIRED,
-            EventType.REMOVED,
-            EventType.UPDATED,
-            EventType.CREATED
-        ).ordered().asynchronous();
-        // ordered() has some performance penalty as compared to unordered(), we can also use synchronous()
-
-        cache = cacheManager.createCache(
-            cacheAlias,
-            CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                EhcacheKey.class,
-                BytesReference.class,
-                ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B, false)
-            ).withService(listenerConfig)
-        );
-    }
-
-    @Override
-    public BytesReference get(IndicesRequestCache.Key key) {
-        // I don't think we need to do the future stuff as the cache is threadsafe
-
-        // if (keystore.contains(key.hashCode()) {
-        long now = System.nanoTime();
-        BytesReference value = null;
-        try {
-            value = cache.get(new EhcacheKey(key));
-        } catch (IOException ignored) { // do smth with this later
-        }
-        double tookTimeMillis = ((double) (System.nanoTime() - now)) / 1000000;
-        getTimeMillisEWMA.addValue(tookTimeMillis);
-        return value;
-        // }
-        // return null;
-    }
-
-    @Override
-    public void put(IndicesRequestCache.Key key, BytesReference value) {
-        // No need to get old value, this is handled by EhcacheEventListener.
-
-        // CheckDataResult policyResult = policy.checkData(value)
-        // if (policyResult.isAccepted()) {
-        try {
-            cache.put(new EhcacheKey(key), value);
-        } catch (IOException ignored) { // do smth with this later
-        }
-        // keystore.add(key.hashCode());
-        // else { do something with policyResult.deniedReason()? }
-        // }
-    }
-
-    @Override
-    public BytesReference computeIfAbsent(IndicesRequestCache.Key key, TieredCacheLoader<IndicesRequestCache.Key, BytesReference> loader)
-        throws Exception {
-        return null; // should not need to fill out, Cache.computeIfAbsent is always used
-    }
-
-    @Override
-    public void invalidate(IndicesRequestCache.Key key) {
-        // keep keystore check to avoid unneeded disk seek
-        // RemovalNotification is handled by EhcacheEventListener
-
-        // if (keystore.contains(key.hashCode()) {
-        try {
-            cache.remove(new EhcacheKey(key));
-        } catch (IOException ignored) { // do smth with this later
-        }
-        // keystore.remove(key.hashCode());
-        // }
-    }
-
-    @Override
-    public BytesReference compute(IndicesRequestCache.Key key, TieredCacheLoader<IndicesRequestCache.Key, BytesReference> loader)
-        throws Exception {
-        return null; // should not need to fill out, Cache.compute is always used
-    }
-
-    @Override
-    public void setRemovalListener(RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener) {
-        this.removalListener = removalListener; // this is passed the spillover strategy, same as on-heap
-    }
-
-    @Override
-    public void invalidateAll() {
-        // can we just wipe the cache and start over? Or do we need to create a bunch of RemovalNotifications?
-    }
-
-    @Override
-    public Iterable<IndicesRequestCache.Key> keys() {
-        // ehcache doesn't provide a method like this, because it might be a huge number of keys that consume all
-        // the memory. Do we want this method for disk tier?
-        return Collections::emptyIterator;
-    }
-
-    @Override
-    public int count() {
-        return (int) count.count();
-    }
-
-    protected void countInc() {
-        count.inc();
-    }
-
-    protected void countDec() {
-        count.dec();
-    }
-
-    @Override
-    public TierType getTierType() {
-        return TierType.DISK;
-    }
-
-    @Override
-    public void onRemoval(RemovalNotification<IndicesRequestCache.Key, BytesReference> notification) {
-        removalListener.onRemoval(notification);
-    }
-
-    @Override
-    public double getTimeMillisEWMA() {
-        return getTimeMillisEWMA.getAverage();
-    }
-
-    public IndicesRequestCache.Key convertEhcacheKeyToOriginal(EhcacheKey eKey) throws IOException {
-        BytesStreamInput is = new BytesStreamInput();
-        byte[] bytes = eKey.getBytes();
-        is.readBytes(bytes, 0, bytes.length);
-        try {
-            return indicesRequestCache.new Key(is);
-        } catch (Exception e) {
-            System.out.println("Was unable to reconstruct EhcacheKey into Key");
-            e.printStackTrace();
-            // actual logging later
-        }
-        return null;
-    }
-
-    @Override
-    public void close() {
-        // Should be called after each test
-        cacheManager.removeCache(cacheAlias);
-        cacheManager.close();
-    }
-
-    public void destroy() throws Exception {
-        // Close the cache and delete any persistent data associated with it
-        // Might also be appropriate after standalone tests
-        cacheManager.close();
-        cacheManager.destroy();
-    }
-}
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java b/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
deleted file mode 100644
index c72fb833a69b6..0000000000000
--- a/server/src/main/java/org/opensearch/indices/EhcacheEventListener.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-import org.opensearch.common.cache.RemovalListener;
-import org.opensearch.common.cache.RemovalNotification;
-import org.opensearch.common.cache.RemovalReason;
-import org.opensearch.core.common.bytes.BytesReference;
-
-import org.ehcache.event.CacheEvent;
-import org.ehcache.event.CacheEventListener;
-import org.ehcache.event.EventType;
-
-public class EhcacheEventListener implements CacheEventListener<EhcacheKey, BytesReference> {
-    // Receives key-value pairs (EhcacheKey, BytesReference), but must transform into (Key, BytesReference)
-    // to send removal notifications
-    private final RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener;
-    private final EhcacheDiskCachingTier tier;
-
-    EhcacheEventListener(RemovalListener<IndicesRequestCache.Key, BytesReference> removalListener, EhcacheDiskCachingTier tier) {
-        this.removalListener = removalListener;
-        this.tier = tier; // needed to handle count changes
-    }
-
-    @Override
-    public void onEvent(CacheEvent<? extends EhcacheKey, ? extends BytesReference> event) {
-        EhcacheKey ehcacheKey = event.getKey();
-        BytesReference oldValue = event.getOldValue();
-        BytesReference newValue = event.getNewValue();
-        EventType eventType = event.getType();
-
-        // handle changing count for the disk tier
-        if (oldValue == null && newValue != null) {
-            tier.countInc();
-        } else if (oldValue != null && newValue == null) {
-            tier.countDec();
-        }
-
-        // handle creating a RemovalReason, unless eventType is CREATED
-        RemovalReason reason;
-        switch (eventType) {
-            case CREATED:
-                return;
-            case EVICTED:
-                reason = RemovalReason.EVICTED; // why is there both RemovalReason.EVICTED and RemovalReason.CAPACITY?
-                break;
-            case EXPIRED:
-            case REMOVED:
-                reason = RemovalReason.INVALIDATED;
-                // this is probably fine for EXPIRED. We use cache.remove() to invalidate keys, but this might overlap with
-                // RemovalReason.EXPLICIT?
-                break;
-            case UPDATED:
-                reason = RemovalReason.REPLACED;
-                break;
-            default:
-                reason = null;
-        }
-        try {
-            IndicesRequestCache.Key key = tier.convertEhcacheKeyToOriginal(ehcacheKey);
-            removalListener.onRemoval(new RemovalNotification<>(key, oldValue, reason));
-        } catch (Exception ignored) {}
-    }
-}
diff --git a/server/src/main/java/org/opensearch/indices/EhcacheKey.java b/server/src/main/java/org/opensearch/indices/EhcacheKey.java
deleted file mode 100644
index f8fa87932d66f..0000000000000
--- a/server/src/main/java/org/opensearch/indices/EhcacheKey.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.io.stream.Writeable;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Arrays;
-
-public class EhcacheKey implements Serializable {
-    // the IndicesRequestCache.Key is not Serializable, but it is Writeable.
-    // We use the output stream's bytes in this wrapper class and implement the appropriate interfaces/methods.
-    private byte[] bytes;
-
-    public EhcacheKey(Writeable key) throws IOException {
-        BytesStreamOutput os = new BytesStreamOutput(); // Should we pass in an expected size? If so, how big?
-        key.writeTo(os);
-        this.bytes = BytesReference.toBytes(os.bytes());
-    }
-
-    public byte[] getBytes() {
-        return this.bytes;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (o == this) {
-            return true;
-        }
-        if (!(o instanceof EhcacheKey)) {
-            return false;
-        }
-        EhcacheKey other = (EhcacheKey) o;
-        return Arrays.equals(this.bytes, other.bytes);
-    }
-
-    @Override
-    public int hashCode() {
-        return Arrays.hashCode(this.bytes);
-    }
-}
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 85832eecbbe62..186a2c98b779b 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -40,6 +40,7 @@
 import org.apache.lucene.util.RamUsageEstimator;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.tier.CacheValue;
 import org.opensearch.common.cache.tier.OnHeapCachingTier;
 import org.opensearch.common.cache.tier.OpenSearchOnHeapCache;
 import org.opensearch.common.cache.tier.TierType;
@@ -144,8 +145,8 @@ void clear(CacheEntity entity) {
     }
 
     @Override
-    public void onMiss(Key key, TierType tierType) {
-        key.entity.onMiss(tierType);
+    public void onMiss(Key key, CacheValue<BytesReference> cacheValue) {
+        key.entity.onMiss(cacheValue);
     }
 
     @Override
@@ -154,8 +155,8 @@ public void onRemoval(RemovalNotification<Key, BytesReference> notification) {
     }
 
     @Override
-    public void onHit(Key key, BytesReference value, TierType tierType) {
-        key.entity.onHit(tierType);
+    public void onHit(Key key, CacheValue<BytesReference> cacheValue) {
+        key.entity.onHit(cacheValue);
     }
 
     @Override
@@ -260,12 +261,12 @@ interface CacheEntity extends Accountable, Writeable {
         /**
          * Called each time this entity has a cache hit.
          */
-        void onHit(TierType tierType);
+        void onHit(CacheValue<BytesReference> cacheValue);
 
         /**
          * Called each time this entity has a cache miss.
          */
-        void onMiss(TierType tierType);
+        void onMiss(CacheValue<BytesReference> cacheValue);
 
         /**
          * Called when this entity instance is removed
diff --git a/server/src/main/java/org/opensearch/indices/OnHeapCachingTier.java b/server/src/main/java/org/opensearch/indices/OnHeapCachingTier.java
deleted file mode 100644
index ea8bd79e1d445..0000000000000
--- a/server/src/main/java/org/opensearch/indices/OnHeapCachingTier.java
+++ /dev/null
@@ -1,11 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-public interface OnHeapCachingTier<K, V> extends CachingTier<K, V> {}
diff --git a/server/src/main/java/org/opensearch/indices/OpenSearchOnHeapCache.java b/server/src/main/java/org/opensearch/indices/OpenSearchOnHeapCache.java
deleted file mode 100644
index cb250edfd7a24..0000000000000
--- a/server/src/main/java/org/opensearch/indices/OpenSearchOnHeapCache.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-import org.opensearch.common.cache.Cache;
-import org.opensearch.common.cache.CacheBuilder;
-import org.opensearch.common.cache.RemovalListener;
-import org.opensearch.common.cache.RemovalNotification;
-import org.opensearch.common.unit.TimeValue;
-
-import java.util.concurrent.ExecutionException;
-import java.util.function.ToLongBiFunction;
-
-public class OpenSearchOnHeapCache<K, V> implements OnHeapCachingTier<K, V>, RemovalListener<K, V> {
-
-    private final Cache<K, V> cache;
-    private RemovalListener<K, V> removalListener;
-
-    private OpenSearchOnHeapCache(long maxWeightInBytes, ToLongBiFunction<K, V> weigher, TimeValue expireAfterAcess) {
-        CacheBuilder<K, V> cacheBuilder = CacheBuilder.<K, V>builder()
-            .setMaximumWeight(maxWeightInBytes)
-            .weigher(weigher)
-            .removalListener(this);
-        if (expireAfterAcess != null) {
-            cacheBuilder.setExpireAfterAccess(expireAfterAcess);
-        }
-        cache = cacheBuilder.build();
-    }
-
-    @Override
-    public void setRemovalListener(RemovalListener<K, V> removalListener) {
-        this.removalListener = removalListener;
-    }
-
-    @Override
-    public void invalidateAll() {
-
-    }
-
-    @Override
-    public Iterable<K> keys() {
-        return this.cache.keys();
-    }
-
-    @Override
-    public int count() {
-        return cache.count();
-    }
-
-    @Override
-    public TierType getTierType() {
-        return TierType.ON_HEAP;
-    }
-
-    @Override
-    public V get(K key) {
-        return cache.get(key);
-    }
-
-    @Override
-    public void put(K key, V value) {
-        cache.put(key, value);
-    }
-
-    @Override
-    public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws ExecutionException {
-        return cache.computeIfAbsent(key, key1 -> loader.load(key));
-    }
-
-    @Override
-    public void invalidate(K key) {
-        cache.invalidate(key);
-    }
-
-    @Override
-    public V compute(K key, TieredCacheLoader<K, V> loader) throws Exception {
-        return cache.compute(key, key1 -> loader.load(key));
-    }
-
-    @Override
-    public void onRemoval(RemovalNotification<K, V> notification) {
-        removalListener.onRemoval(notification);
-    }
-
-    public static class Builder<K, V> {
-        private long maxWeightInBytes;
-
-        private ToLongBiFunction<K, V> weigher;
-
-        private TimeValue expireAfterAcess;
-
-        public Builder() {}
-
-        public Builder<K, V> setMaximumWeight(long sizeInBytes) {
-            this.maxWeightInBytes = sizeInBytes;
-            return this;
-        }
-
-        public Builder<K, V> setWeigher(ToLongBiFunction<K, V> weigher) {
-            this.weigher = weigher;
-            return this;
-        }
-
-        public Builder<K, V> setExpireAfterAccess(TimeValue expireAfterAcess) {
-            this.expireAfterAcess = expireAfterAcess;
-            return this;
-        }
-
-        public OpenSearchOnHeapCache<K, V> build() {
-            return new OpenSearchOnHeapCache<K, V>(maxWeightInBytes, weigher, expireAfterAcess);
-        }
-    }
-}
diff --git a/server/src/main/java/org/opensearch/indices/TierType.java b/server/src/main/java/org/opensearch/indices/TierType.java
deleted file mode 100644
index 900df2fce57c5..0000000000000
--- a/server/src/main/java/org/opensearch/indices/TierType.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-public enum TierType {
-
-    ON_HEAP("on_heap"),
-    DISK("disk");
-
-    private final String stringValue;
-
-    TierType(String stringValue) {
-        // Associate each TierType with a string representation, for use in API responses and elsewhere
-        this.stringValue = stringValue;
-    }
-
-    public String getStringValue() {
-        return this.stringValue;
-    }
-}
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheEventListener.java b/server/src/main/java/org/opensearch/indices/TieredCacheEventListener.java
deleted file mode 100644
index 084ac5a57e0d3..0000000000000
--- a/server/src/main/java/org/opensearch/indices/TieredCacheEventListener.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-import org.opensearch.common.cache.RemovalNotification;
-
-public interface TieredCacheEventListener<K, V> {
-
-    void onMiss(K key, TierType tierType);
-
-    void onRemoval(RemovalNotification<K, V> notification);
-
-    void onHit(K key, V value, TierType tierType);
-
-    void onCached(K key, V value, TierType tierType);
-}
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
deleted file mode 100644
index 4816f94f7d619..0000000000000
--- a/server/src/main/java/org/opensearch/indices/TieredCacheHandler.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-public interface TieredCacheHandler<K, V> {
-
-    V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception;
-
-    V get(K key);
-
-    void invalidate(K key);
-
-    void invalidateAll();
-
-    long count();
-
-    CachingTier<K, V> getOnHeapCachingTier();
-
-    void closeDiskTier();
-
-    double diskGetTimeMillisEWMA();
-}
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheLoader.java b/server/src/main/java/org/opensearch/indices/TieredCacheLoader.java
deleted file mode 100644
index f6bb1a74e973e..0000000000000
--- a/server/src/main/java/org/opensearch/indices/TieredCacheLoader.java
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-public interface TieredCacheLoader<K, V> {
-    V load(K key) throws Exception;
-
-    boolean isLoaded();
-}
diff --git a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java b/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
deleted file mode 100644
index f02a193c7d354..0000000000000
--- a/server/src/main/java/org/opensearch/indices/TieredCacheSpilloverStrategyHandler.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.indices;
-
-import org.opensearch.common.cache.RemovalListener;
-import org.opensearch.common.cache.RemovalNotification;
-import org.opensearch.common.cache.RemovalReason;
-import org.opensearch.core.common.io.stream.Writeable;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
-import java.util.function.Function;
-
-/**
- *
- * @param <K>
- * @param <V>
- */
-public class TieredCacheSpilloverStrategyHandler<K extends Writeable, V> implements TieredCacheHandler<K, V>, RemovalListener<K, V> {
-
-    private final OnHeapCachingTier<K, V> onHeapCachingTier;
-    private final DiskCachingTier<K, V> diskCachingTier;
-    private final TieredCacheEventListener<K, V> tieredCacheEventListener;
-
-    /**
-     * Maintains caching tiers in order of get calls.
-     */
-    private final List<CachingTier<K, V>> cachingTierList;
-
-    private TieredCacheSpilloverStrategyHandler(
-        OnHeapCachingTier<K, V> onHeapCachingTier,
-        DiskCachingTier<K, V> diskCachingTier,
-        TieredCacheEventListener<K, V> tieredCacheEventListener
-    ) {
-        this.onHeapCachingTier = Objects.requireNonNull(onHeapCachingTier);
-        this.diskCachingTier = Objects.requireNonNull(diskCachingTier);
-        this.tieredCacheEventListener = tieredCacheEventListener;
-        this.cachingTierList = Arrays.asList(onHeapCachingTier, diskCachingTier);
-        setRemovalListeners();
-    }
-
-    @Override
-    public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception {
-        CacheValue<V> cacheValue = getValueFromTierCache().apply(key);
-        if (cacheValue == null) {
-            // Add the value to the onHeap cache. Any items if evicted will be moved to lower tier
-            V value = onHeapCachingTier.compute(key, loader);
-            tieredCacheEventListener.onCached(key, value, TierType.ON_HEAP);
-            return value;
-        }
-        return cacheValue.value;
-    }
-
-    @Override
-    public V get(K key) {
-        CacheValue<V> cacheValue = getValueFromTierCache().apply(key);
-        if (cacheValue == null) {
-            return null;
-        }
-        return cacheValue.value;
-    }
-
-    @Override
-    public void invalidate(K key) {
-        // TODO
-    }
-
-    @Override
-    public void invalidateAll() {
-        for (CachingTier<K, V> cachingTier : cachingTierList) {
-            cachingTier.invalidateAll();
-        }
-    }
-
-    @Override
-    public long count() {
-        long totalCount = 0;
-        for (CachingTier<K, V> cachingTier : cachingTierList) {
-            totalCount += cachingTier.count();
-        }
-        return totalCount;
-    }
-
-    public long count(TierType tierType) {
-        for (CachingTier<K, V> cachingTier : cachingTierList) {
-            if (cachingTier.getTierType() == tierType) {
-                return cachingTier.count();
-            }
-        }
-        return -1L;
-    }
-
-    @Override
-    public void onRemoval(RemovalNotification<K, V> notification) {
-        if (RemovalReason.EVICTED.equals(notification.getRemovalReason())) {
-            switch (notification.getTierType()) {
-                case ON_HEAP:
-                    diskCachingTier.put(notification.getKey(), notification.getValue());
-
-                    break;
-                default:
-                    break;
-            }
-        }
-        tieredCacheEventListener.onRemoval(notification);
-    }
-
-    @Override
-    public CachingTier<K, V> getOnHeapCachingTier() {
-        return this.onHeapCachingTier;
-    }
-
-    public DiskCachingTier<K, V> getDiskCachingTier() { // change to CachingTier after debug
-        return this.diskCachingTier;
-    }
-
-    private void setRemovalListeners() {
-        for (CachingTier<K, V> cachingTier : cachingTierList) {
-            cachingTier.setRemovalListener(this);
-        }
-    }
-
-    private Function<K, CacheValue<V>> getValueFromTierCache() {
-        return key -> {
-            for (CachingTier<K, V> cachingTier : cachingTierList) {
-                V value = cachingTier.get(key);
-                if (value != null) {
-                    tieredCacheEventListener.onHit(key, value, cachingTier.getTierType());
-                    return new CacheValue<>(value, cachingTier.getTierType());
-                }
-                tieredCacheEventListener.onMiss(key, cachingTier.getTierType());
-            }
-            return null;
-        };
-    }
-
-    @Override
-    public void closeDiskTier() {
-        diskCachingTier.close();
-    }
-
-    @Override
-    public double diskGetTimeMillisEWMA() {
-        return diskCachingTier.getTimeMillisEWMA();
-    }
-
-    public static class CacheValue<V> {
-        V value;
-        TierType source;
-
-        CacheValue(V value, TierType source) {
-            this.value = value;
-            this.source = source;
-        }
-    }
-
-    public static class Builder<K extends Writeable, V> {
-        private OnHeapCachingTier<K, V> onHeapCachingTier;
-        private DiskCachingTier<K, V> diskCachingTier;
-        private TieredCacheEventListener<K, V> tieredCacheEventListener;
-
-        public Builder() {}
-
-        public Builder<K, V> setOnHeapCachingTier(OnHeapCachingTier<K, V> onHeapCachingTier) {
-            this.onHeapCachingTier = onHeapCachingTier;
-            return this;
-        }
-
-        public Builder<K, V> setOnDiskCachingTier(DiskCachingTier<K, V> diskCachingTier) {
-            this.diskCachingTier = diskCachingTier;
-            return this;
-        }
-
-        public Builder<K, V> setTieredCacheEventListener(TieredCacheEventListener<K, V> tieredCacheEventListener) {
-            this.tieredCacheEventListener = tieredCacheEventListener;
-            return this;
-        }
-
-        public TieredCacheSpilloverStrategyHandler<K, V> build() {
-            return new TieredCacheSpilloverStrategyHandler<K, V>(
-                this.onHeapCachingTier,
-                this.diskCachingTier,
-                this.tieredCacheEventListener
-            );
-        }
-    }
-
-}
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java b/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
index bb7a22cc26037..bf4b46440733d 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
@@ -340,8 +340,8 @@ class MockTieredCacheEventListener<K, V> implements TieredCacheEventListener<K,
         }
 
         @Override
-        public void onMiss(K key, TierType tierType) {
-            enumMap.get(tierType).missCount.inc();
+        public void onMiss(K key, CacheValue<V> cacheValue) {
+            enumMap.get(cacheValue.getSource()).missCount.inc();
         }
 
         @Override
@@ -352,8 +352,8 @@ public void onRemoval(RemovalNotification<K, V> notification) {
         }
 
         @Override
-        public void onHit(K key, V value, TierType tierType) {
-            enumMap.get(tierType).hitCount.inc();
+        public void onHit(K key, CacheValue<V> cacheValue) {
+            enumMap.get(cacheValue.getSource()).hitCount.inc();
         }
 
         @Override
diff --git a/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java b/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
index 6992b8a441c0a..8b57b094e347f 100644
--- a/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
+++ b/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
@@ -11,7 +11,6 @@
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.BytesStreamInput;
-import org.opensearch.indices.TierType;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.HashMap;
diff --git a/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java b/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java
index 5dd4eb504ec2f..accd7a29efb43 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesServiceCloseTests.java
@@ -37,6 +37,7 @@
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.tier.CacheValue;
 import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.concurrent.OpenSearchExecutors;
@@ -342,10 +343,10 @@ public Object getCacheIdentity() {
             }
 
             @Override
-            public void onHit(TierType tierType) {}
+            public void onHit(CacheValue<BytesReference> cacheValue) {}
 
             @Override
-            public void onMiss(TierType tierType) {}
+            public void onMiss(CacheValue<BytesReference> cacheValue) {}
 
             @Override
             public void onRemoval(RemovalNotification<Key, BytesReference> notification) {}

From 900230d975c12d1a1a7f6276e1c8e2e92d982413 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 9 Nov 2023 16:15:30 -0800
Subject: [PATCH 054/120] Finished cachevalue rework + disk stats
 implementation

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../index/cache/request/DiskStatsHolder.java  |  99 -----------
 .../cache/request/OnHeapStatsHolder.java      |  37 -----
 .../cache/request/RequestCacheStats.java      |  64 ++++++--
 .../cache/request/ShardRequestCache.java      | 154 ++++++++++++++++--
 .../index/cache/request/StatsHolder.java      |  12 +-
 .../indices/IndicesRequestCache.java          |  20 ++-
 .../org/opensearch/bootstrap/security.policy  |   8 +
 .../tier/EhCacheDiskCachingTierTests.java     |  13 +-
 ...redCacheSpilloverStrategyServiceTests.java |   8 +-
 .../cache/request/RequestCacheStatsTests.java |  30 +++-
 .../cache/request/ShardRequestCacheTests.java |  40 +++++
 .../indices/IndicesRequestCacheTests.java     | 130 +--------------
 12 files changed, 302 insertions(+), 313 deletions(-)
 delete mode 100644 server/src/main/java/org/opensearch/index/cache/request/DiskStatsHolder.java
 delete mode 100644 server/src/main/java/org/opensearch/index/cache/request/OnHeapStatsHolder.java
 create mode 100644 server/src/test/java/org/opensearch/index/cache/request/ShardRequestCacheTests.java

diff --git a/server/src/main/java/org/opensearch/index/cache/request/DiskStatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/DiskStatsHolder.java
deleted file mode 100644
index 094b358333908..0000000000000
--- a/server/src/main/java/org/opensearch/index/cache/request/DiskStatsHolder.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.index.cache.request;
-
-import org.opensearch.common.cache.tier.DiskTierRequestStats;
-import org.opensearch.common.cache.tier.TierRequestStats;
-import org.opensearch.common.cache.tier.TierType;
-import org.opensearch.common.metrics.CounterMetric;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.xcontent.XContentBuilder;
-
-import java.io.IOException;
-
-public class DiskStatsHolder extends StatsHolder {
-    final CounterMetric totalGetTime;
-    final CounterMetric totalDiskReaches; // Number of times a get() has actually reached the disk
-    public DiskStatsHolder() {
-        super();
-        this.totalGetTime = new CounterMetric();
-        this.totalDiskReaches = new CounterMetric();
-    }
-
-    public DiskStatsHolder(long memorySize, long evictions, long hitCount, long missCount, long entries, long totalGetTime, long totalDiskReaches) {
-        super(memorySize, evictions, hitCount, missCount, entries);
-        this.totalGetTime = new CounterMetric();
-        this.totalGetTime.inc(totalGetTime);
-        this.totalDiskReaches = new CounterMetric();
-        this.totalDiskReaches.inc(totalDiskReaches);
-    }
-
-    public DiskStatsHolder(StreamInput in) throws IOException {
-        this(
-            in.readVLong(),
-            in.readVLong(),
-            in.readVLong(),
-            in.readVLong(),
-            in.readVLong(),
-            in.readVLong(),
-            in.readVLong()
-        );
-    }
-
-    public long getTotalGetTime() {
-        return totalGetTime.count();
-    }
-
-    public long getTotalDiskReaches() {
-        return totalDiskReaches.count();
-    }
-
-    @Override
-    public void addRequestStats(TierRequestStats stats) {
-        assert stats.getTierType() == TierType.DISK;
-        DiskTierRequestStats diskStats = (DiskTierRequestStats) stats;
-        // TODO: Should this instead be accomplished with generics somehow
-        if (diskStats.getRequestReachedDisk()) {
-            this.totalDiskReaches.inc();
-            this.totalGetTime.inc(diskStats.getRequestGetTimeNanos());
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeVLong(totalGetTime.count());
-        out.writeVLong(totalDiskReaches.count());
-    }
-
-    @Override
-    public void add(StatsHolder stats) {
-        assert stats.getClass() == DiskStatsHolder.class;
-        super.add(stats);
-        DiskStatsHolder diskStats = (DiskStatsHolder) stats;
-        totalDiskReaches.inc(diskStats.totalDiskReaches.count());
-        totalGetTime.inc(diskStats.totalGetTime.count());
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        super.toXContent(builder, params);
-        builder.field(Fields.TOTAL_GET_TIME, getTotalGetTime());
-        builder.field(Fields.TOTAL_DISK_REACHES, getTotalDiskReaches());
-        return builder;
-    }
-
-    static final class Fields {
-        static final String TOTAL_GET_TIME = "total_get_time_nanos";
-        static final String TOTAL_DISK_REACHES = "total_disk_reaches";
-    }
-
-
-}
diff --git a/server/src/main/java/org/opensearch/index/cache/request/OnHeapStatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/OnHeapStatsHolder.java
deleted file mode 100644
index 5ef11f7850c5d..0000000000000
--- a/server/src/main/java/org/opensearch/index/cache/request/OnHeapStatsHolder.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.index.cache.request;
-
-import org.opensearch.common.cache.tier.TierRequestStats;
-import org.opensearch.core.common.io.stream.StreamInput;
-
-import java.io.IOException;
-
-
-/**
- * This class accumulates stats for a single shard, for the on-heap tier.
- * For now, on-heap tier has no unique stats, but future stats would be added here.
- */
-public class OnHeapStatsHolder extends StatsHolder {
-    public OnHeapStatsHolder() {
-        super();
-    }
-
-    public OnHeapStatsHolder(long memorySize, long evictions, long hitCount, long missCount, long entries) {
-        super(memorySize, evictions, hitCount, missCount, entries);
-    }
-
-    public OnHeapStatsHolder(StreamInput in) throws IOException {
-        super(in);
-    }
-
-    @Override
-    public void addRequestStats(TierRequestStats stats) {}
-
-}
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index 1907541621968..e3b1e7667d640 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -32,7 +32,9 @@
 
 package org.opensearch.index.cache.request;
 
+import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
+import org.opensearch.common.cache.tier.TierRequestStats;
 import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -42,6 +44,7 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -52,44 +55,64 @@
  */
 public class RequestCacheStats implements Writeable, ToXContentFragment {
 
-    private Map<String, StatsHolder> map = new HashMap<>(){{
-        for (TierType tierType : TierType.values())
-        {
+    private Map<String, StatsHolder> defaultStatsMap = new HashMap<>(){{
+        for (TierType tierType : TierType.values()) {
             put(tierType.getStringValue(), new StatsHolder());
             // Every possible tier type must have counters, even if they are disabled. Then the counters report 0
         }}
     };
 
+     private Map<String, ShardRequestCache.TierStatsAccumulator> tierSpecificStatsMap = new HashMap<>(){{
+        put(TierType.ON_HEAP.getStringValue(), new ShardRequestCache.OnHeapStatsAccumulator());
+        put(TierType.DISK.getStringValue(), new ShardRequestCache.DiskStatsAccumulator());
+        //checkTierSpecificMap(); // might yell, idk if the map is done until the block ends
+     }};
+
     public RequestCacheStats() {}
 
     public RequestCacheStats(StreamInput in) throws IOException {
         if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
-            this.map = in.readMap(StreamInput::readString, StatsHolder::new);
+            this.defaultStatsMap = in.readMap(StreamInput::readString, StatsHolder::new);
+            // Manually fill the tier-specific stats map
+            this.tierSpecificStatsMap = new HashMap<>();
+            tierSpecificStatsMap.put(TierType.ON_HEAP.getStringValue(), new ShardRequestCache.OnHeapStatsAccumulator(in));
+            tierSpecificStatsMap.put(TierType.DISK.getStringValue(), new ShardRequestCache.DiskStatsAccumulator(in));
         } else {
             // objects from earlier versions only contain on-heap info, and do not have entries info
             long memorySize = in.readVLong();
             long evictions = in.readVLong();
             long hitCount = in.readVLong();
             long missCount = in.readVLong();
-            this.map.put(TierType.ON_HEAP.getStringValue(), new StatsHolder(memorySize, evictions, hitCount, missCount, 0));
+            this.defaultStatsMap.put(TierType.ON_HEAP.getStringValue(), new StatsHolder(memorySize, evictions, hitCount, missCount, 0));
         }
+        checkTierSpecificMap();
     }
 
-    public RequestCacheStats(Map<TierType, StatsHolder> inputMap) {
+    public RequestCacheStats(Map<TierType, StatsHolder> defaultMap, Map<TierType, ShardRequestCache.TierStatsAccumulator> tierSpecificMap) {
         // Create a RequestCacheStats with multiple tiers' statistics
-        for (TierType tierType : inputMap.keySet()) {
-            map.put(tierType.getStringValue(), inputMap.get(tierType));
+        // The input maps don't have to have all tiers statistics available
+        for (TierType tierType : defaultMap.keySet()) {
+            defaultStatsMap.put(tierType.getStringValue(), defaultMap.get(tierType));
+        }
+        for (TierType tierType : tierSpecificMap.keySet()) {
+            tierSpecificStatsMap.put(tierType.getStringValue(), tierSpecificMap.get(tierType));
         }
+        checkTierSpecificMap();
     }
 
     public void add(RequestCacheStats stats) {
-        for (String tier : stats.map.keySet()) {
-            map.get(tier).add(stats.map.get(tier));
+        for (TierType tierType : TierType.values()) {
+            defaultStatsMap.get(tierType.getStringValue()).add(stats.defaultStatsMap.get(tierType.getStringValue()));
+            tierSpecificStatsMap.get(tierType.getStringValue()).add(stats.tierSpecificStatsMap.get(tierType.getStringValue()));
         }
     }
 
     private StatsHolder getTierStats(TierType tierType) {
-        return map.get(tierType.getStringValue());
+        return defaultStatsMap.get(tierType.getStringValue());
+    }
+
+    ShardRequestCache.TierStatsAccumulator getTierSpecificStats(TierType tierType) { // pkg-private for testing
+        return tierSpecificStatsMap.get(tierType.getStringValue());
     }
 
     public long getMemorySizeInBytes(TierType tierType) {
@@ -145,10 +168,13 @@ public long getEntries() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
-            out.writeMap(this.map, StreamOutput::writeString, (o, v) -> v.writeTo(o)); // ?
+            out.writeMap(this.defaultStatsMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); // ?
+            // Manually write tier-specific stats map
+            tierSpecificStatsMap.get(TierType.ON_HEAP.getStringValue()).writeTo(out);
+            tierSpecificStatsMap.get(TierType.DISK.getStringValue()).writeTo(out);
         } else {
             // Write only on-heap values, and don't write entries metric
-            StatsHolder heapStats = map.get(TierType.ON_HEAP.getStringValue());
+            StatsHolder heapStats = defaultStatsMap.get(TierType.ON_HEAP.getStringValue());
             out.writeVLong(heapStats.getMemorySize());
             out.writeVLong(heapStats.getEvictions());
             out.writeVLong(heapStats.getHitCount());
@@ -161,12 +187,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject(Fields.REQUEST_CACHE_STATS);
         // write on-heap stats outside of tiers object
         getTierStats(TierType.ON_HEAP).toXContent(builder, params);
+        getTierSpecificStats(TierType.ON_HEAP).toXContent(builder, params);
         builder.startObject(Fields.TIERS);
         for (TierType tierType : TierType.values()) { // fixed order
             if (tierType != TierType.ON_HEAP) {
                 String tier = tierType.getStringValue();
                 builder.startObject(tier);
-                map.get(tier).toXContent(builder, params);
+                defaultStatsMap.get(tier).toXContent(builder, params);
+                tierSpecificStatsMap.get(tier).toXContent(builder, params);
                 builder.endObject();
             }
         }
@@ -175,6 +203,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
+    private void checkTierSpecificMap() {
+        for (TierType tierType : TierType.values()) {
+            if (tierSpecificStatsMap.get(tierType.getStringValue()) == null) {
+                throw new OpenSearchException("Missing TierStatsAccumulator for TierType " + tierType.getStringValue());
+            }
+        }
+    }
+
     /**
      * Fields used for parsing and toXContent
      *
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index 2a9ddf9c33618..6e7f4d335ecd1 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -33,11 +33,21 @@
 package org.opensearch.index.cache.request;
 
 import org.apache.lucene.util.Accountable;
+import org.opensearch.OpenSearchException;
 import org.opensearch.common.cache.tier.CacheValue;
+import org.opensearch.common.cache.tier.DiskTierRequestStats;
+import org.opensearch.common.cache.tier.OnHeapTierRequestStats;
+import org.opensearch.common.cache.tier.TierRequestStats;
 import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
 
+import java.io.IOException;
 import java.util.EnumMap;
 
 /**
@@ -47,34 +57,47 @@
  */
 public final class ShardRequestCache {
 
-    private EnumMap<TierType, StatsHolder> statsHolder = new EnumMap<>(TierType.class);
+    // Holds stats common to all tiers
+    private final EnumMap<TierType, StatsHolder> defaultStatsHolder = new EnumMap<>(TierType.class);
+
+    // Holds tier-specific stats
+    private final EnumMap<TierType, TierStatsAccumulator> tierSpecificStatsHolder = new EnumMap<>(TierType.class);
 
     public ShardRequestCache() {
+        tierSpecificStatsHolder.put(TierType.ON_HEAP, new OnHeapStatsAccumulator());
+        tierSpecificStatsHolder.put(TierType.DISK, new DiskStatsAccumulator());
         for (TierType tierType : TierType.values()) {
-            statsHolder.put(tierType, new StatsHolder());
+            defaultStatsHolder.put(tierType, new StatsHolder());
+            if (tierSpecificStatsHolder.get(tierType) == null) {
+                throw new OpenSearchException("Missing TierStatsAccumulator for TierType " + tierType.getStringValue());
+            }
         }
     }
 
     public RequestCacheStats stats() {
-        return new RequestCacheStats(statsHolder);
+        return new RequestCacheStats(defaultStatsHolder, tierSpecificStatsHolder);
     }
 
     public void onHit(CacheValue<BytesReference> cacheValue) {
-        statsHolder.get(cacheValue.getSource()).hitCount.inc();
+        TierType source = cacheValue.getSource();
+        defaultStatsHolder.get(source).hitCount.inc();
+        tierSpecificStatsHolder.get(source).addRequestStats(cacheValue.getStats());
     }
 
     public void onMiss(CacheValue<BytesReference> cacheValue) {
-        statsHolder.get(cacheValue.getSource()).missCount.inc();
+        TierType source = cacheValue.getSource();
+        defaultStatsHolder.get(cacheValue.getSource()).missCount.inc();
+        tierSpecificStatsHolder.get(source).addRequestStats(cacheValue.getStats());
     }
 
     public void onCached(Accountable key, BytesReference value, TierType tierType) {
-        statsHolder.get(tierType).totalMetric.inc(key.ramBytesUsed() + value.ramBytesUsed());
-        statsHolder.get(tierType).entries.inc();
+        defaultStatsHolder.get(tierType).totalMetric.inc(key.ramBytesUsed() + value.ramBytesUsed());
+        defaultStatsHolder.get(tierType).entries.inc();
     }
 
     public void onRemoval(Accountable key, BytesReference value, boolean evicted, TierType tierType) {
         if (evicted) {
-            statsHolder.get(tierType).evictionsMetric.inc();
+            defaultStatsHolder.get(tierType).evictionsMetric.inc();
         }
         long dec = 0;
         if (key != null) {
@@ -83,7 +106,118 @@ public void onRemoval(Accountable key, BytesReference value, boolean evicted, Ti
         if (value != null) {
             dec += value.ramBytesUsed();
         }
-        statsHolder.get(tierType).totalMetric.dec(dec);
-        statsHolder.get(tierType).entries.dec();
+        defaultStatsHolder.get(tierType).totalMetric.dec(dec);
+        defaultStatsHolder.get(tierType).entries.dec();
+    }
+
+    /**
+     * An abstract whose extending classes accumulate tier-specific stats.
+     * All extending classes should provide a constructor like TierStatsAccumulator(StreamInput in)
+     * as well as a no-argument constructor
+     * @param <S> The tier-specific implementation of TierRequestStats to use
+     */
+    static abstract class TierStatsAccumulator<S extends TierRequestStats> implements Writeable, ToXContentFragment {
+        /**
+         * Add new stats from a single request to this holder.
+         * @param stats
+         */
+        public abstract void addRequestStats(S stats);
+
+        /**
+         * Add the stats from another TierStatsHolder to this TierStatsHolder.
+         * Used when combining stats across multiple shards.
+         * @param other The other TierStatsHolder.
+         */
+        public abstract void add(TierStatsAccumulator<S> other);
+    }
+
+    /**
+     * This class accumulates on-heap-tier-specific stats for a single shard.
+     * For now, on-heap tier has no unique stats, but future stats would be added here.
+     */
+    static class OnHeapStatsAccumulator extends TierStatsAccumulator<OnHeapTierRequestStats> {
+        OnHeapStatsAccumulator() {}
+        OnHeapStatsAccumulator(StreamInput in) {}
+        @Override
+        public void addRequestStats(OnHeapTierRequestStats stats) {}
+
+        @Override
+        public void add(TierStatsAccumulator<OnHeapTierRequestStats> other) {}
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {}
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder;
+        }
+    }
+
+    /**
+     * This class accumulates disk-tier-specific stats for a single shard.
+     */
+    static class DiskStatsAccumulator extends TierStatsAccumulator<DiskTierRequestStats> {
+        final CounterMetric totalGetTime;
+        final CounterMetric totalDiskReaches; // Number of times a get() has actually reached the disk
+        public DiskStatsAccumulator() {
+            this.totalGetTime = new CounterMetric();
+            this.totalDiskReaches = new CounterMetric();
+        }
+
+        public DiskStatsAccumulator(long totalGetTime, long totalDiskReaches) {
+            this.totalGetTime = new CounterMetric();
+            this.totalGetTime.inc(totalGetTime);
+            this.totalDiskReaches = new CounterMetric();
+            this.totalDiskReaches.inc(totalDiskReaches);
+        }
+
+        public DiskStatsAccumulator(StreamInput in) throws IOException {
+            this(
+                in.readVLong(),
+                in.readVLong()
+            );
+        }
+
+        public long getTotalGetTime() {
+            return totalGetTime.count();
+        }
+
+        public long getTotalDiskReaches() {
+            return totalDiskReaches.count();
+        }
+
+        @Override
+        public void addRequestStats(DiskTierRequestStats stats) {
+            if (stats.getRequestReachedDisk()) {
+                this.totalDiskReaches.inc();
+                this.totalGetTime.inc(stats.getRequestGetTimeNanos());
+            }
+        }
+
+        @Override
+        public void add(TierStatsAccumulator<DiskTierRequestStats> other) {
+            assert other.getClass() == DiskStatsAccumulator.class;
+            DiskStatsAccumulator castOther = (DiskStatsAccumulator) other;
+            this.totalDiskReaches.inc(castOther.totalDiskReaches.count());
+            this.totalGetTime.inc(castOther.totalGetTime.count());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVLong(totalGetTime.count());
+            out.writeVLong(totalDiskReaches.count());
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(Fields.TOTAL_GET_TIME, getTotalGetTime());
+            builder.field(Fields.TOTAL_DISK_REACHES, getTotalDiskReaches());
+            return builder;
+        }
+
+        static final class Fields { // Used for field names in API response
+            static final String TOTAL_GET_TIME = "total_get_time_nanos";
+            static final String TOTAL_DISK_REACHES = "total_disk_reaches";
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
index 7596da018e842..300689cf09dff 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
@@ -20,20 +20,20 @@
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * The basic StatsHolder class, which accumulates shard-level stats that are common across all tiers.
  * Used in ShardRequestCache. Extending classes also handle tier-specific stats for each tier.
  */
-public abstract class StatsHolder implements Serializable, Writeable, ToXContentFragment {
+public class StatsHolder implements Serializable, Writeable, ToXContentFragment {
     final CounterMetric totalMetric;
     final CounterMetric evictionsMetric;
     final CounterMetric hitCount;
     final CounterMetric missCount;
     final CounterMetric entries;
 
-    public static final Map<TierType, >
-
     public StatsHolder() {
         this.totalMetric = new CounterMetric();
         this.evictionsMetric = new CounterMetric();
@@ -115,10 +115,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(RequestCacheStats.Fields.ENTRIES, getEntries());
         return builder;
     }
-
-    /**
-     * Used to add stats for a single request (get call) to existing stats
-     * @param stats Stats for a single request
-     */
-    public abstract void addRequestStats(TierRequestStats stats);
 }
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 186a2c98b779b..21fc131b42402 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -41,6 +41,7 @@
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.tier.CacheValue;
+import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
 import org.opensearch.common.cache.tier.OnHeapCachingTier;
 import org.opensearch.common.cache.tier.OpenSearchOnHeapCache;
 import org.opensearch.common.cache.tier.TierType;
@@ -127,10 +128,27 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             (k, v) -> k.ramBytesUsed() + v.ramBytesUsed()
         ).setMaximumWeight(sizeInBytes).setExpireAfterAccess(expire).build();
 
+        // enabling this for testing purposes. Remove/tweak!!
+        //long CACHE_SIZE_IN_BYTES = 1000000L;
+        //String SETTING_PREFIX = "indices.request.cache";
+
+        /*EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
+            .setKeyType(Key.class)
+            .setValueType(BytesReference.class)
+            .setExpireAfterAccess(TimeValue.MAX_VALUE)
+            .setSettings(settings)
+            .setThreadPoolAlias("ehcacheTest")
+            .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+            .setStoragePath("/tmp")
+            .setSettingPrefix(SETTING_PREFIX)
+            .build();*/
+
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
         tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>().setOnHeapCachingTier(
             openSearchOnHeapCache
-        ).setTieredCacheEventListener(this).build();
+        )
+            //.setOnDiskCachingTier(ehcacheDiskTier)
+            .setTieredCacheEventListener(this).build();
         this.indicesService = indicesService;
     }
 
diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy
index 6dcc50be95ccb..1e588742f64c4 100644
--- a/server/src/main/resources/org/opensearch/bootstrap/security.policy
+++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy
@@ -191,4 +191,12 @@ grant {
   // For ehcache
   permission java.lang.RuntimePermission "createClassLoader";
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
+
+  // Testing, remove later
+  //permission java.io.FilePermission "/tmp", "read";
+  //permission java.io.FilePermission "/tmp", "write";
+  //permission java.io.FilePermission "/tmp", "delete";
+  //permission java.io.FilePermission "/tmp/-", "read";
+  //permission java.io.FilePermission "/tmp/-", "write";
+  //permission java.io.FilePermission "/tmp/-", "delete";
 };
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
index 804f236264daa..e3efc6eece5d7 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
@@ -50,8 +50,11 @@ public void testBasicGetAndPut() throws IOException {
                 ehCacheDiskCachingTierNew.put(entry.getKey(), entry.getValue());
             }
             for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
-                String value = ehCacheDiskCachingTierNew.get(entry.getKey());
-                assertEquals(entry.getValue(), value);
+                CacheValue<String> value = ehCacheDiskCachingTierNew.get(entry.getKey());
+                assertEquals(entry.getValue(), value.value);
+                assertEquals(TierType.DISK, value.getSource());
+                assertTrue(((DiskTierRequestStats) value.getStats()).getRequestReachedDisk());
+                assertTrue(((DiskTierRequestStats) value.getStats()).getRequestGetTimeNanos() > 0);
             }
             ehCacheDiskCachingTierNew.close();
         }
@@ -91,8 +94,8 @@ public void testConcurrentPut() throws Exception {
             phaser.arriveAndAwaitAdvance(); // Will trigger parallel puts above.
             countDownLatch.await(); // Wait for all threads to finish
             for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
-                String value = ehCacheDiskCachingTierNew.get(entry.getKey());
-                assertEquals(entry.getValue(), value);
+                CacheValue<String> value = ehCacheDiskCachingTierNew.get(entry.getKey());
+                assertEquals(entry.getValue(), value.value);
             }
             ehCacheDiskCachingTierNew.close();
         }
@@ -129,7 +132,7 @@ public void testEhcacheParallelGets() throws Exception {
             for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
                 threads[j] = new Thread(() -> {
                     phaser.arriveAndAwaitAdvance();
-                    assertEquals(entry.getValue(), ehCacheDiskCachingTierNew.get(entry.getKey()));
+                    assertEquals(entry.getValue(), ehCacheDiskCachingTierNew.get(entry.getKey()).value);
                     countDownLatch.countDown();
                 });
                 threads[j].start();
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java b/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
index bf4b46440733d..a85d82118ff66 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
@@ -254,8 +254,8 @@ class MockOnHeapCacheTier<K, V> implements OnHeapCachingTier<K, V>, RemovalListe
         }
 
         @Override
-        public V get(K key) {
-            return this.onHeapCacheTier.get(key);
+        public CacheValue<V> get(K key) {
+            return new CacheValue(this.onHeapCacheTier.get(key), TierType.ON_HEAP, new OnHeapTierRequestStats());
         }
 
         @Override
@@ -381,8 +381,8 @@ class MockDiskCachingTier<K, V> implements DiskCachingTier<K, V>, RemovalListene
         }
 
         @Override
-        public V get(K key) {
-            return this.diskTier.get(key);
+        public CacheValue<V> get(K key) {
+            return new CacheValue<>(this.diskTier.get(key), TierType.DISK, new DiskTierRequestStats(0L, true));
         }
 
         @Override
diff --git a/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java b/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
index 8b57b094e347f..b481cf762f9ec 100644
--- a/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
+++ b/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.index.cache.request;
 
+import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.BytesStreamInput;
@@ -22,27 +23,35 @@ public void testConstructorsAndAdd() throws Exception {
         for (TierType tierType : TierType.values()) {
             assertTierState(emptyStats, tierType, 0, 0, 0, 0, 0);
         }
+        assertDiskStatsState(emptyStats, 0, 0);
         Map<TierType, StatsHolder> testHeapMap = new HashMap<>();
         testHeapMap.put(TierType.ON_HEAP, new StatsHolder(1, 2, 3, 4, 5));
-        RequestCacheStats heapOnlyStats = new RequestCacheStats(testHeapMap);
+        Map<TierType, ShardRequestCache.TierStatsAccumulator> tierSpecificMap = new HashMap<>();
+        tierSpecificMap.put(TierType.DISK, new ShardRequestCache.DiskStatsAccumulator(6, 7));
+        RequestCacheStats heapAndSpecificOnlyStats = new RequestCacheStats(testHeapMap, tierSpecificMap);
         for (TierType tierType : TierType.values()) {
             if (tierType == TierType.ON_HEAP) {
-                assertTierState(heapOnlyStats, tierType, 1, 2, 3, 4, 5);
+                assertTierState(heapAndSpecificOnlyStats, tierType, 1, 2, 3, 4, 5);
             } else {
-                assertTierState(heapOnlyStats, tierType, 0, 0, 0, 0, 0);
+                assertTierState(heapAndSpecificOnlyStats, tierType, 0, 0, 0, 0, 0);
             }
         }
+        assertDiskStatsState(heapAndSpecificOnlyStats, 6, 7);
 
         Map<TierType, StatsHolder> testBothTiersMap = new HashMap<>();
         testBothTiersMap.put(TierType.ON_HEAP, new StatsHolder(11, 12, 13, 14, 15));
         testBothTiersMap.put(TierType.DISK, new StatsHolder(6, 7, 8, 9, 10));
-        RequestCacheStats bothTiersStats = new RequestCacheStats(testBothTiersMap);
+        Map<TierType, ShardRequestCache.TierStatsAccumulator> newTierSpecificMap = new HashMap<>();
+        newTierSpecificMap.put(TierType.ON_HEAP, new ShardRequestCache.OnHeapStatsAccumulator());
+        newTierSpecificMap.put(TierType.DISK, new ShardRequestCache.DiskStatsAccumulator(8, 9));
+        RequestCacheStats bothTiersStats = new RequestCacheStats(testBothTiersMap, newTierSpecificMap);
         assertTierState(bothTiersStats, TierType.ON_HEAP, 11, 12, 13, 14, 15);
         assertTierState(bothTiersStats, TierType.DISK, 6, 7, 8, 9, 10);
 
-        bothTiersStats.add(heapOnlyStats);
+        bothTiersStats.add(heapAndSpecificOnlyStats);
         assertTierState(bothTiersStats, TierType.ON_HEAP, 12, 14, 16, 18, 20);
         assertTierState(bothTiersStats, TierType.DISK, 6, 7, 8, 9, 10);
+        assertDiskStatsState(bothTiersStats, 14, 16);
     }
 
     public void testSerialization() throws Exception {
@@ -52,13 +61,17 @@ public void testSerialization() throws Exception {
         Map<TierType, StatsHolder> testMap = new HashMap<>();
         testMap.put(TierType.ON_HEAP, new StatsHolder(11, 12, 13, 14, 15));
         testMap.put(TierType.DISK, new StatsHolder(6, 7, 8, 9, 10));
-        RequestCacheStats stats = new RequestCacheStats(testMap);
+        Map<TierType, ShardRequestCache.TierStatsAccumulator> tierSpecificMap = new HashMap<>();
+        tierSpecificMap.put(TierType.ON_HEAP, new ShardRequestCache.OnHeapStatsAccumulator());
+        tierSpecificMap.put(TierType.DISK, new ShardRequestCache.DiskStatsAccumulator(20, 21));
+        RequestCacheStats stats = new RequestCacheStats(testMap, tierSpecificMap);
         stats.writeTo(os);
         BytesStreamInput is = new BytesStreamInput(BytesReference.toBytes(os.bytes()));
         RequestCacheStats deserialized = new RequestCacheStats(is);
 
         assertTierState(deserialized, TierType.ON_HEAP, 11, 12, 13, 14, 15);
         assertTierState(deserialized, TierType.DISK, 6, 7, 8, 9, 10);
+        assertDiskStatsState(deserialized, 20, 21);
     }
 
     private void assertTierState(
@@ -76,4 +89,9 @@ private void assertTierState(
         assertEquals(missCount, stats.getMissCount(tierType));
         assertEquals(entries, stats.getEntries(tierType));
     }
+
+    private void assertDiskStatsState(RequestCacheStats stats, long totalGetTime, long totalDiskReaches) {
+        assertEquals(totalGetTime, ((ShardRequestCache.DiskStatsAccumulator) stats.getTierSpecificStats(TierType.DISK)).getTotalGetTime());
+        assertEquals(totalDiskReaches, ((ShardRequestCache.DiskStatsAccumulator) stats.getTierSpecificStats(TierType.DISK)).getTotalDiskReaches());
+    }
 }
diff --git a/server/src/test/java/org/opensearch/index/cache/request/ShardRequestCacheTests.java b/server/src/test/java/org/opensearch/index/cache/request/ShardRequestCacheTests.java
new file mode 100644
index 0000000000000..8b97924343152
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/cache/request/ShardRequestCacheTests.java
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.cache.request;
+
+
+import org.opensearch.common.cache.tier.DiskTierRequestStats;
+import org.opensearch.test.OpenSearchTestCase;
+
+public class ShardRequestCacheTests extends OpenSearchTestCase {
+    // Serialization and getter logic is implicitly tested in RequestCacheStatsTests.java,
+    // in this file, check logic for StatsHolder.TierStatsAccumulator implementations
+
+    public void testInit() throws Exception {
+        ShardRequestCache src = new ShardRequestCache();
+        RequestCacheStats rcs = src.stats();
+    }
+    public void testDiskStatsAccumulator() throws Exception {
+        ShardRequestCache.DiskStatsAccumulator acc = new ShardRequestCache.DiskStatsAccumulator();
+        DiskTierRequestStats reachedDiskReqStats = new DiskTierRequestStats(145L, true);
+        acc.addRequestStats(reachedDiskReqStats);
+        assertEquals(1, acc.getTotalDiskReaches());
+        assertEquals(145, acc.getTotalGetTime());
+        DiskTierRequestStats noDiskReqStats = new DiskTierRequestStats(391392L, false);
+        acc.addRequestStats(noDiskReqStats);
+        assertEquals(1, acc.getTotalDiskReaches());
+        assertEquals(145, acc.getTotalGetTime());
+
+        ShardRequestCache.DiskStatsAccumulator other = new ShardRequestCache.DiskStatsAccumulator();
+        other.addRequestStats(new DiskTierRequestStats(1L, true));
+        acc.add(other);
+        assertEquals(146, acc.getTotalGetTime());
+        assertEquals(2, acc.getTotalDiskReaches());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 89168e37191fa..18ec013711f22 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -51,7 +51,6 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.bytes.AbstractBytesReference;
-import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -66,7 +65,6 @@
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 
 import java.io.IOException;
-import java.io.Serializable;
 import java.util.Arrays;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -126,126 +124,6 @@ public void testBasicOperationsCache() throws Exception {
 
         IOUtils.close(reader, writer, dir, cache);
         assertEquals(0, cache.numRegisteredCloseListeners());
-        cache.closeDiskTier();
-    }
-
-    public void testAddDirectToEhcache() throws Exception {
-        // this test is for debugging serialization issues and can eventually be removed
-        // Put breakpoints at line 260 of AbstractOffHeapStore to catch serialization errors
-        // that would otherwise fail silently
-        ShardRequestCache requestCacheStats = new ShardRequestCache();
-        Settings.Builder settingsBuilder = Settings.builder();
-        long heapSizeBytes = 1000;
-        settingsBuilder.put("indices.requests.cache.size", new ByteSizeValue(heapSizeBytes));
-        IndicesRequestCache cache = new IndicesRequestCache(settingsBuilder.build(), getInstanceFromNode(IndicesService.class));
-
-        // set up a key
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-        writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        AtomicBoolean indexShard = new AtomicBoolean(true);
-        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        String rKey = ((OpenSearchDirectoryReader) reader).getDelegatingCacheHelper().getDelegatingCacheKey().getId().toString();
-        IndicesRequestCache.Key key = cache.new Key(entity, termBytes, rKey);
-
-        BytesReference value = new BytesArray(new byte[] { 0 });
-        cache.tieredCacheHandler.getDiskCachingTier().put(key, value);
-
-        BytesReference res = cache.tieredCacheHandler.getDiskCachingTier().get(key);
-        assertEquals(value, res);
-        assertEquals(1, cache.tieredCacheHandler.count(TierType.DISK));
-
-        IOUtils.close(reader, writer, dir, cache);
-        cache.closeDiskTier();
-    }
-
-    public void testSpillover() throws Exception {
-        // fill the on-heap cache until we spill over
-        ShardRequestCache requestCacheStats = new ShardRequestCache();
-        Settings.Builder settingsBuilder = Settings.builder();
-        long heapSizeBytes = 1000; // each of these queries is 131 bytes, so we can fit 7 in the heap cache
-        int heapKeySize = 131;
-        int maxNumInHeap = 1000 / heapKeySize;
-        settingsBuilder.put("indices.requests.cache.size", new ByteSizeValue(heapSizeBytes));
-        IndicesRequestCache cache = new IndicesRequestCache(settingsBuilder.build(), getInstanceFromNode(IndicesService.class));
-
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-        writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        AtomicBoolean indexShard = new AtomicBoolean(true);
-
-        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
-        Loader loader = new Loader(reader, 0);
-        System.out.println("On-heap cache size at start = " + requestCacheStats.stats().getMemorySizeInBytes());
-        BytesReference[] termBytesArr = new BytesReference[maxNumInHeap + 1];
-
-        for (int i = 0; i < maxNumInHeap + 1; i++) {
-            TermQueryBuilder termQuery = new TermQueryBuilder("id", String.valueOf(i));
-            BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-            String rKey = ((OpenSearchDirectoryReader) reader).getDelegatingCacheHelper().getDelegatingCacheKey().getId().toString();
-            termBytesArr[i] = termBytes;
-            BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
-        }
-        // get value from disk cache, the first key should have been evicted
-        BytesReference firstValue = cache.getOrCompute(entity, loader, reader, termBytesArr[0]);
-
-        assertEquals(maxNumInHeap * heapKeySize, requestCacheStats.stats().getMemorySizeInBytes());
-        // TODO: disk weight bytes
-        assertEquals(1, requestCacheStats.stats().getEvictions());
-        assertEquals(1, requestCacheStats.stats().getHitCount(TierType.DISK));
-        assertEquals(maxNumInHeap + 1, requestCacheStats.stats().getMissCount(TierType.DISK));
-        assertEquals(0, requestCacheStats.stats().getHitCount());
-        assertEquals(maxNumInHeap + 2, requestCacheStats.stats().getMissCount());
-        assertEquals(maxNumInHeap, cache.tieredCacheHandler.count(TierType.ON_HEAP));
-        assertEquals(1, cache.tieredCacheHandler.count(TierType.DISK));
-
-        // get a value from heap cache, second key should still be there
-        BytesReference secondValue = cache.getOrCompute(entity, loader, reader, termBytesArr[1]);
-        // get the value on disk cache again
-        BytesReference firstValueAgain = cache.getOrCompute(entity, loader, reader, termBytesArr[0]);
-
-        assertEquals(1, requestCacheStats.stats().getEvictions());
-        assertEquals(2, requestCacheStats.stats().getHitCount(TierType.DISK));
-        assertEquals(maxNumInHeap + 1, requestCacheStats.stats().getMissCount(TierType.DISK));
-        assertEquals(1, requestCacheStats.stats().getHitCount());
-        assertEquals(maxNumInHeap + 3, requestCacheStats.stats().getMissCount());
-        assertEquals(maxNumInHeap, cache.tieredCacheHandler.count(TierType.ON_HEAP));
-        assertEquals(1, cache.tieredCacheHandler.count(TierType.DISK));
-
-        IOUtils.close(reader, writer, dir, cache);
-        cache.closeDiskTier();
-    }
-
-    public void testDiskGetTimeEWMA() throws Exception {
-        ShardRequestCache requestCacheStats = new ShardRequestCache();
-        Settings.Builder settingsBuilder = Settings.builder();
-        long heapSizeBytes = 0; // skip directly to disk cache
-        settingsBuilder.put("indices.requests.cache.size", new ByteSizeValue(heapSizeBytes));
-        IndicesRequestCache cache = new IndicesRequestCache(settingsBuilder.build(), getInstanceFromNode(IndicesService.class));
-
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-        writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        AtomicBoolean indexShard = new AtomicBoolean(true);
-
-        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
-        Loader loader = new Loader(reader, 0);
-
-        for (int i = 0; i < 50; i++) {
-            TermQueryBuilder termQuery = new TermQueryBuilder("id", String.valueOf(i));
-            BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-            BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
-            // on my machine get time EWMA converges to ~0.025 ms, but it does have an SSD
-            assertTrue(cache.tieredCacheHandler.diskGetTimeMillisEWMA() > 0);
-        }
-
-        IOUtils.close(reader, writer, dir, cache);
-        cache.closeDiskTier();
     }
 
     public void testCacheDifferentReaders() throws Exception {
@@ -340,7 +218,6 @@ public void testCacheDifferentReaders() throws Exception {
 
         IOUtils.close(secondReader, writer, dir, cache);
         assertEquals(0, cache.numRegisteredCloseListeners());
-        cache.closeDiskTier();
     }
 
     public void testEviction() throws Exception {
@@ -370,7 +247,6 @@ public void testEviction() throws Exception {
             assertEquals("bar", value2.streamInput().readString());
             size = requestCacheStats.stats().getMemorySize();
             IOUtils.close(reader, secondReader, writer, dir, cache);
-            cache.closeDiskTier();
         }
         IndicesRequestCache cache = new IndicesRequestCache(
             Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), size.getBytes() + 1 + "b").build(),
@@ -408,7 +284,6 @@ public void testEviction() throws Exception {
         assertEquals(2, cache.count());
         assertEquals(1, requestCacheStats.stats().getEvictions());
         IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
-        cache.closeDiskTier();
     }
 
     public void testClearAllEntityIdentity() throws Exception {
@@ -456,7 +331,6 @@ public void testClearAllEntityIdentity() throws Exception {
         assertEquals("baz", value3.streamInput().readString());
 
         IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
-        cache.closeDiskTier();
 
     }
 
@@ -562,7 +436,6 @@ public void testInvalidate() throws Exception {
 
         IOUtils.close(reader, writer, dir, cache);
         assertEquals(0, cache.numRegisteredCloseListeners());
-        cache.closeDiskTier();
     }
 
     public void testEqualsKey() throws IOException {
@@ -622,6 +495,7 @@ public void testSerializationDeserializationOfCacheKey() throws Exception {
     }
 
     private class TestBytesReference extends AbstractBytesReference {
+
         int dummyValue;
 
         TestBytesReference(int dummyValue) {
@@ -671,7 +545,7 @@ public boolean isFragment() {
         }
     }
 
-    private class TestEntity extends AbstractIndexShardCacheEntity implements Serializable {
+    private class TestEntity extends AbstractIndexShardCacheEntity {
         private final AtomicBoolean standInForIndexShard;
         private final ShardRequestCache shardRequestCache;
 

From 07a2e4ba250c2af5d438fc6fa5a401ca3a0ffc61 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Fri, 10 Nov 2023 16:37:14 -0800
Subject: [PATCH 055/120] Simplified RBM size estimator

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../cache/tier/keystore/KeyLookupStore.java   |   2 +-
 .../cache/tier/keystore/KeyStoreStats.java    |   4 +-
 .../tier/keystore/RBMIntKeyLookupStore.java   |  41 ++++--
 .../cache/tier/keystore/RBMSizeEstimator.java | 137 ------------------
 .../keystore/RBMIntKeyLookupStoreTests.java   |  64 ++++----
 5 files changed, 63 insertions(+), 185 deletions(-)
 delete mode 100644 server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
index a08174b5b46d3..dc2b7a4ba1234 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
@@ -103,7 +103,7 @@ public interface KeyLookupStore<T> {
 
     /**
      * Returns an estimate of the store's memory usage.
-     * @return The memory usage, in MB
+     * @return The memory usage
      */
     long getMemorySizeInBytes();
 
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java
index b5c95b134990c..ab3055a81d4c9 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java
@@ -22,17 +22,15 @@ public class KeyStoreStats {
     protected CounterMetric numAddAttempts;
     protected CounterMetric numCollisions;
     protected boolean guaranteesNoFalseNegatives;
-    protected final int maxNumEntries;
     protected AtomicBoolean atCapacity;
     protected CounterMetric numRemovalAttempts;
     protected CounterMetric numSuccessfulRemovals;
 
-    protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) {
+    protected KeyStoreStats(long memSizeCapInBytes) {
         this.size = new CounterMetric();
         this.numAddAttempts = new CounterMetric();
         this.numCollisions = new CounterMetric();
         this.memSizeCapInBytes = memSizeCapInBytes;
-        this.maxNumEntries = maxNumEntries;
         this.atCapacity = new AtomicBoolean(false);
         this.numRemovalAttempts = new CounterMetric();
         this.numSuccessfulRemovals = new CounterMetric();
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index 9dba9e8165d40..e7370558da80b 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -34,7 +34,6 @@
 
 import org.opensearch.common.metrics.CounterMetric;
 
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.concurrent.locks.Lock;
@@ -77,10 +76,14 @@ public int getValue() {
     protected RoaringBitmap rbm;
     private HashMap<Integer, CounterMetric> collidedIntCounters;
     private HashMap<Integer, HashSet<Integer>> removalSets;
-    protected RBMSizeEstimator sizeEstimator;
     protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
     protected final Lock readLock = lock.readLock();
     protected final Lock writeLock = lock.writeLock();
+    private long mostRecentByteEstimate;
+    private final int REFRESH_SIZE_EST_INTERVAL = 10000;
+    // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms,
+    // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store
+    // in terms of storage impact
 
     // Default constructor sets modulo = 2^28
     public RBMIntKeyLookupStore(long memSizeCapInBytes) {
@@ -89,18 +92,11 @@ public RBMIntKeyLookupStore(long memSizeCapInBytes) {
 
     public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBytes) {
         this.modulo = moduloValue.getValue();
-        sizeEstimator = new RBMSizeEstimator(modulo);
-        this.stats = new KeyStoreStats(memSizeCapInBytes, calculateMaxNumEntries(memSizeCapInBytes));
+        this.stats = new KeyStoreStats(memSizeCapInBytes);
         this.rbm = new RoaringBitmap();
         this.collidedIntCounters = new HashMap<>();
         this.removalSets = new HashMap<>();
-    }
-
-    protected int calculateMaxNumEntries(long memSizeCapInBytes) {
-        if (memSizeCapInBytes == 0) {
-            return Integer.MAX_VALUE;
-        }
-        return sizeEstimator.getNumEntriesFromSizeInBytes(memSizeCapInBytes);
+        this.mostRecentByteEstimate = 0L;
     }
 
     private final int transform(int value) {
@@ -125,7 +121,11 @@ public boolean add(Integer value) {
             return false;
         }
         stats.numAddAttempts.inc();
-        if (stats.size.count() == stats.maxNumEntries) {
+
+        if (getSize() % REFRESH_SIZE_EST_INTERVAL == 0) {
+            mostRecentByteEstimate = getMemorySizeInBytes();
+        }
+        if (getMemorySizeCapInBytes() > 0 && mostRecentByteEstimate > getMemorySizeCapInBytes()) {
             stats.atCapacity.set(true);
             return false;
         }
@@ -273,9 +273,24 @@ public boolean isCollision(Integer value1, Integer value2) {
         return transform(value1) == transform(value2);
     }
 
+     static double getRBMSizeMultiplier(int numEntries, int modulo) {
+        double x = Math.log10((double) numEntries / modulo);
+        if (x < -5) {
+            return 7.0;
+        }
+        if (x < -2.75) {
+            return -2.5 * x - 5.5;
+        }
+        if (x <= 0) {
+            return -3.0 / 22.0 * x + 1;
+        }
+        return 1;
+    }
+
     @Override
     public long getMemorySizeInBytes() {
-        return sizeEstimator.getSizeInBytes((int) stats.size.count()); // + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size());
+        double multiplier = getRBMSizeMultiplier((int) stats.size.count(), modulo);
+        return (long) (rbm.getSizeInBytes() * multiplier);
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java
deleted file mode 100644
index e07c645bad0cf..0000000000000
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-package org.opensearch.common.cache.tier.keystore;
-
-import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
-
-/**
- * A class used to estimate roaring bitmap memory sizes (and hash set sizes).
- * Values based on experiments with adding randomly distributed integers, which matches the use case for KeyLookupStore.
- * In this use case, true values are much higher than an RBM's self-reported size, especially for small RBMs: see
- * https://github.com/RoaringBitmap/RoaringBitmap/issues/257
- */
-public class RBMSizeEstimator {
-    public static final int BYTES_IN_MB = 1048576;
-    public static final double HASHSET_MEM_SLOPE = 6.46 * Math.pow(10, -5);
-    protected final double slope;
-    protected final double bufferMultiplier;
-    protected final double intercept;
-
-    RBMSizeEstimator(int modulo) {
-        double[] memValues = calculateMemoryCoefficients(modulo);
-        this.bufferMultiplier = memValues[0];
-        this.slope = memValues[1];
-        this.intercept = memValues[2];
-    }
-
-    public static double[] calculateMemoryCoefficients(int modulo) {
-        // Sets up values to help estimate RBM size given a modulo
-        // Returns an array of {bufferMultiplier, slope, intercept}
-
-        double modifiedModulo;
-        if (modulo == 0) {
-            modifiedModulo = 32.0;
-        } else {
-            modifiedModulo = Math.log(modulo) / Math.log(2);
-        }
-        // we "round up" the modulo to the nearest tested value
-        double highCutoff = 29.001; // Floating point makes 29 not work
-        double mediumCutoff = 28.0;
-        double lowCutoff = 26.0;
-        double bufferMultiplier = 1.0;
-        double slope;
-        double intercept;
-        if (modifiedModulo > highCutoff) {
-            // modulo > 2^29
-            bufferMultiplier = 1.2;
-            slope = 0.637;
-            intercept = 3.091;
-        } else if (modifiedModulo > mediumCutoff) {
-            // 2^29 >= modulo > 2^28
-            slope = 0.619;
-            intercept = 2.993;
-        } else if (modifiedModulo > lowCutoff) {
-            // 2^28 >= modulo > 2^26
-            slope = 0.614;
-            intercept = 2.905;
-        } else {
-            slope = 0.628;
-            intercept = 2.603;
-        }
-        return new double[] { bufferMultiplier, slope, intercept };
-    }
-
-    public long getSizeInBytes(int numEntries) {
-        // Based on a linear fit in log-log space, so that we minimize the error as a proportion rather than as
-        // an absolute value. Should be within ~50% of the true value at worst, and should overestimate rather
-        // than underestimate the memory usage
-        return (long) ((long) Math.pow(numEntries, slope) * (long) Math.pow(10, intercept) * bufferMultiplier);
-    }
-
-    public int getNumEntriesFromSizeInBytes(long sizeInBytes) {
-        // This function has some precision issues especially when composed with its inverse:
-        // numEntries = getNumEntriesFromSizeInBytes(getSizeInBytes(numEntries))
-        // In this case the result can be off by up to a couple percent
-        // However, this shouldn't really matter as both functions are based on memory estimates with higher errors than a couple percent
-        // and this composition won't happen outside of tests
-        return (int) Math.pow(sizeInBytes / (bufferMultiplier * Math.pow(10, intercept)), 1 / slope);
-
-    }
-
-    public static long getSizeInBytesWithModulo(int numEntries, int modulo) {
-        double[] memValues = calculateMemoryCoefficients(modulo);
-        return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]);
-    }
-
-    public static long getSizeInBytesWithModuloValue(int numEntries, RBMIntKeyLookupStore.KeystoreModuloValue moduloValue) {
-        double[] memValues = calculateMemoryCoefficients(moduloValue.getValue());
-        return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]);
-    }
-
-    public static int getNumEntriesFromSizeInBytesWithModulo(long sizeInBytes, int modulo) {
-        double[] memValues = calculateMemoryCoefficients(modulo);
-        return (int) Math.pow(sizeInBytes / (memValues[0] * Math.pow(10, memValues[2])), 1 / memValues[1]);
-    }
-
-    protected static long convertMBToBytes(double valMB) {
-        return (long) (valMB * BYTES_IN_MB);
-    }
-
-    protected static double convertBytesToMB(long valBytes) {
-        return (double) valBytes / BYTES_IN_MB;
-    }
-
-    protected static long getHashsetMemSizeInBytes(int numEntries) {
-        return convertMBToBytes(HASHSET_MEM_SLOPE * numEntries);
-    }
-}
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index 0e9d63a9731a5..a6d076c8250ad 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -32,10 +32,9 @@
 package org.opensearch.common.cache.tier.keystore;
 
 import org.opensearch.common.Randomness;
-import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
-import org.opensearch.common.cache.tier.keystore.RBMSizeEstimator;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.test.OpenSearchTestCase;
+import org.roaringbitmap.RoaringBitmap;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -45,8 +44,10 @@
 import java.util.concurrent.ThreadPoolExecutor;
 
 public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase {
+
+    final int BYTES_IN_MB = 1048576;
     public void testInit() {
-        long memCap = 100 * RBMSizeEstimator.BYTES_IN_MB;
+        long memCap = 100 * BYTES_IN_MB;
         RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(memCap);
         assertEquals(0, kls.getSize());
         assertEquals(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_EIGHT.getValue(), kls.modulo);
@@ -129,16 +130,35 @@ public void testAddingDuplicates() throws Exception {
     }
 
     public void testMemoryCapBlocksAdd() throws Exception {
+        // Now that we're using a modified version of rbm.getSizeInBytes(), which doesn't provide an inverse function,
+        // we have to test filling just an RBM with random test values first so that we can get the resulting memory cap limit
+        // to use with our modified size estimate.
+        // This is much noisier so the precision is lower.
+
+        // It is necessary to use randomly distributed integers for both parts of this test, as we would do with hashes in the cache,
+        // as that's what our size estimator is designed for.
+        // If we add a run of integers, our size estimator is not valid, especially for small RBMs.
+
+        int[] maxEntriesArr = new int[] { 1342000, 100000, 3000000};
+        long[] rbmReportedSizes = new long[4];
+        Random rand = Randomness.get();
+        for (int j = 0; j < maxEntriesArr.length; j++)  {
+            RoaringBitmap rbm = new RoaringBitmap();
+            for (int i = 0; i < maxEntriesArr[j]; i++) {
+                rbm.add(rand.nextInt());
+            }
+            rbmReportedSizes[j] = rbm.getSizeInBytes();
+        }
         RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE;
-        for (int maxEntries : new int[] { 2342000, 1000, 100000 }) {
-            long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue);
+        for (int i = 0; i < maxEntriesArr.length; i++) {
+            double multiplier = RBMIntKeyLookupStore.getRBMSizeMultiplier(maxEntriesArr[i], moduloValue.getValue());
+            long memSizeCapInBytes = (long) (rbmReportedSizes[i] * multiplier);
+            //long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue);
             RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes);
-            for (int j = 0; j < maxEntries + 1000; j++) {
-                kls.add(j);
+            for (int j = 0; j < maxEntriesArr[i] + 5000; j++) {
+                kls.add(rand.nextInt());
             }
-            assertTrue(Math.abs(maxEntries - kls.getSize()) < (double) maxEntries / 25);
-            // exact cap varies a small amount bc of floating point, especially when we use bytes instead of MB for calculations
-            // precision gets much worse when we compose the two functions, as we do here, but this wouldn't happen in an actual use case
+            assertTrue(Math.abs(maxEntriesArr[i] - kls.getSize()) < (double) maxEntriesArr[i] / 10);
         }
     }
 
@@ -205,7 +225,7 @@ public void testConcurrency() throws Exception {
     }
 
     public void testRemoveNoCollisions() throws Exception {
-        long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB;
+        long memCap = 100L * BYTES_IN_MB;
         int numToAdd = 195000;
         RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, memCap);
         // there should be no collisions for sequential positive numbers up to modulo
@@ -222,7 +242,7 @@ public void testRemoveNoCollisions() throws Exception {
 
     public void testRemoveWithCollisions() throws Exception {
         int modulo = (int) Math.pow(2, 26);
-        long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB;
+        long memCap = 100L * BYTES_IN_MB;
         RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX, memCap);
         for (int i = 0; i < 10; i++) {
             kls.add(i);
@@ -278,24 +298,6 @@ public void testNullInputs() throws Exception {
         assertEquals(4, kls.getSize());
     }
 
-    public void testMemoryCapValueInitialization() {
-        RBMIntKeyLookupStore.KeystoreModuloValue[] mods = RBMIntKeyLookupStore.KeystoreModuloValue.values();
-        double[] expectedMultipliers = new double[] { 1.2, 1.2, 1, 1, 1 };
-        double[] expectedSlopes = new double[] { 0.637, 0.637, 0.619, 0.614, 0.629 };
-        double[] expectedIntercepts = new double[] { 3.091, 3.091, 2.993, 2.905, 2.603 };
-        long memSizeCapInBytes = (long) 100.0 * RBMSizeEstimator.BYTES_IN_MB;
-        double delta = 0.01;
-        for (int i = 0; i < mods.length; i++) {
-            RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = mods[i];
-            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes);
-            assertEquals(kls.stats.memSizeCapInBytes, kls.getMemorySizeCapInBytes(), 1.0);
-            assertEquals(expectedMultipliers[i], kls.sizeEstimator.bufferMultiplier, delta);
-            assertEquals(expectedSlopes[i], kls.sizeEstimator.slope, delta);
-            assertEquals(expectedIntercepts[i], kls.sizeEstimator.intercept, delta);
-        }
-
-    }
-
     public void testRemovalLogic() throws Exception {
         RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX;
         int modulo = moduloValue.getValue();
@@ -395,6 +397,6 @@ public void testRemovalLogicWithHashCollision() throws Exception {
         assertTrue(removalSet.contains(k1));
         assertEquals(1, numCollisions.count());
         assertTrue(kls.contains(k1));
-        assertTrue(kls.contains(k2)); 
+        assertTrue(kls.contains(k2));
     }
 }

From 2f884c2baecf4a7916347a6073620cd0fefbe06f Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 13 Nov 2023 10:14:14 -0800
Subject: [PATCH 056/120] addressed sagar's comments

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../common/cache/tier/EhCacheDiskCachingTier.java  | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index 4adfcd731c6ef..ff42ac510231b 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -44,14 +44,6 @@
 import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
 
 /**
- * This ehcache disk caching tier uses its value serializer outside ehcache.
- * Values are transformed to byte[] outside ehcache and then ehcache uses its bundled byte[] serializer.
- * The key serializer you pass to this class produces a byte[]. This serializer is passed to a wrapper which
- * implements Ehcache's serializer implementation and produces a BytesBuffer. The wrapper instance is then passed to ehcache.
- * This is done because to get keys on a disk tier, ehcache internally checks the equals() method of the serializer,
- * but ALSO requires newKey.equals(storedKey) (this isn't documented), which is the case for ByteBuffer but not byte[].
- * This limitation means that the key serializer must preserve the class of the key before/after serialization,
- * but the value serializer does not have to do this.
  * @param <K> The key type of cache entries
  * @param <V> The value type of cache entries
  */
@@ -293,7 +285,8 @@ public void onEvent(CacheEvent<? extends K, ? extends byte[]> event) {
                             new RemovalNotification<>(
                                 event.getKey(),
                                 valueSerializer.deserialize(event.getOldValue()),
-                                RemovalReason.EVICTED
+                                RemovalReason.EVICTED,
+                                TierType.DISK
                             )
                         )
                     );
@@ -310,7 +303,8 @@ public void onEvent(CacheEvent<? extends K, ? extends byte[]> event) {
                             new RemovalNotification<>(
                                 event.getKey(),
                                 valueSerializer.deserialize(event.getOldValue()),
-                                RemovalReason.INVALIDATED
+                                RemovalReason.INVALIDATED,
+                                TierType.DISK
                             )
                         )
                     );

From 4e8ba0b8419e87078b5e7c72c910c62c325cb1a5 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 13 Nov 2023 11:40:59 -0800
Subject: [PATCH 057/120] Fixed broken test

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../common/cache/tier/EhCacheDiskCachingTierTests.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
index 866f0598abede..194af9098fb15 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
@@ -75,7 +75,7 @@ public void testBasicGetAndPutBytesReference() throws Exception {
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setSettings(settings)
                 .setThreadPoolAlias("ehcacheTest")
-                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES * 2) // bigger so no evictions happen
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
                 .setSettingPrefix(SETTING_PREFIX)
                 .setKeySerializer(new StringSerializer())

From 09748eb8d9cbe5efc75848fefd2c828a2241b70d Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 13 Nov 2023 13:39:53 -0800
Subject: [PATCH 058/120] Re-added and tested serializer for IRC key

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../indices/IRCKeyWriteableSerializer.java    | 61 +++++++++++++++++++
 .../indices/IndicesRequestCache.java          |  9 ++-
 .../tier/EhCacheDiskCachingTierTests.java     |  2 +-
 .../IRCKeyWriteableSerializerTests.java       | 57 +++++++++++++++++
 4 files changed, 125 insertions(+), 4 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
 create mode 100644 server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java

diff --git a/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java b/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
new file mode 100644
index 0000000000000..63a3447d481c0
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.OpenSearchException;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+import org.opensearch.common.cache.tier.Serializer;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * This class serializes the IndicesRequestCache.Key using its writeTo method.
+ */
+public class IRCKeyWriteableSerializer implements Serializer<IndicesRequestCache.Key, byte[]> {
+
+    IndicesRequestCache irc;
+    public IRCKeyWriteableSerializer(IndicesRequestCache irc) {
+        this.irc = irc;
+    }
+    @Override
+    public byte[] serialize(IndicesRequestCache.Key object) {
+        try {
+            BytesStreamOutput os = new BytesStreamOutput();
+            object.writeTo(os);
+            return BytesReference.toBytes(os.bytes());
+        } catch (IOException e) {
+            throw new OpenSearchException(e);
+        }
+    }
+
+    @Override
+    public IndicesRequestCache.Key deserialize(byte[] bytes) {
+        try {
+            BytesStreamInput is = new BytesStreamInput(bytes, 0, bytes.length);
+            return irc.new Key(is);
+        } catch (IOException e) {
+            throw new OpenSearchException(e);
+        }
+    }
+
+    @Override
+    public boolean equals(IndicesRequestCache.Key object, byte[] bytes) {
+        // Deserialization is much slower than serialization for keys of order 1 KB,
+        // while time to serialize is fairly constant (per byte)
+        if (bytes.length < 5000) {
+            return Arrays.equals(serialize(object), bytes);
+        } else {
+            return object.equals(deserialize(bytes));
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 013d6ae5be549..441229d9e3893 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -40,6 +40,7 @@
 import org.apache.lucene.util.RamUsageEstimator;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.tier.BytesReferenceSerializer;
 import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
 import org.opensearch.common.cache.tier.CacheValue;
 import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
@@ -140,8 +141,8 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             }
         };
         // enabling this for testing purposes. Remove/tweak!!
-        //long CACHE_SIZE_IN_BYTES = 1000000L;
-        //String SETTING_PREFIX = "indices.request.cache";
+        long CACHE_SIZE_IN_BYTES = 1000000L;
+        String SETTING_PREFIX = "indices.request.cache";
 
         /*EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
             .setKeyType(Key.class)
@@ -152,6 +153,8 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
             .setStoragePath("/tmp")
             .setSettingPrefix(SETTING_PREFIX)
+            .setKeySerializer(new IRCKeyWriteableSerializer(this))
+            .setValueSerializer(new BytesReferenceSerializer())
             .build();*/
 
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
@@ -316,7 +319,7 @@ interface CacheEntity extends Accountable, Writeable {
      *
      * @opensearch.internal
      */
-    class Key implements Accountable, Writeable {
+     class Key implements Accountable, Writeable {
         private final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Key.class);
 
         public final CacheEntity entity; // use as identity equality
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
index a0f11b7c70a2b..45928b2f20610 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
@@ -97,7 +97,7 @@ public void testBasicGetAndPutBytesReference() throws Exception {
                 ehCacheDiskCachingTier.put(entry.getKey(), entry.getValue());
             }
             for (Map.Entry<String, BytesReference> entry : keyValueMap.entrySet()) {
-                BytesReference value = ehCacheDiskCachingTier.get(entry.getKey());
+                BytesReference value = ehCacheDiskCachingTier.get(entry.getKey()).value;
                 assertEquals(entry.getValue(), value);
             }
             ehCacheDiskCachingTier.close();
diff --git a/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java b/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
new file mode 100644
index 0000000000000..c18250bb6bec2
--- /dev/null
+++ b/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.indices;
+
+import org.opensearch.common.Randomness;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.index.IndexService;
+import org.opensearch.index.shard.IndexShard;
+import org.opensearch.test.OpenSearchSingleNodeTestCase;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.UUID;
+
+public class IRCKeyWriteableSerializerTests extends OpenSearchSingleNodeTestCase {
+
+    public void testSerializer() throws Exception {
+        ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndicesRequestCache irc = new IndicesRequestCache(Settings.EMPTY, indicesService, dummyClusterSettings);
+        IndexService indexService = createIndex("test");
+        IndexShard indexShard = indexService.getShardOrNull(0);
+        IndicesService.IndexShardCacheEntity entity = indicesService.new IndexShardCacheEntity(indexShard);
+        IRCKeyWriteableSerializer ser = new IRCKeyWriteableSerializer(irc);
+
+        int NUM_KEYS = 1000;
+        int[] valueLengths = new int[]{ 1000, 6000 }; // test both branches in equals()
+        Random rand = Randomness.get();
+        for (int valueLength: valueLengths) {
+            for (int i = 0; i < NUM_KEYS; i++) {
+                IndicesRequestCache.Key key = getRandomIRCKey(valueLength, rand, irc, entity);
+                byte[] serialized = ser.serialize(key);
+                assertTrue(ser.equals(key, serialized));
+                IndicesRequestCache.Key deserialized = ser.deserialize(serialized);
+                assertTrue(key.equals(deserialized));
+            }
+        }
+    }
+    private IndicesRequestCache.Key getRandomIRCKey(int valueLength, Random random, IndicesRequestCache irc, IndicesService.IndexShardCacheEntity entity) {
+        byte[] value = new byte[valueLength];
+        for (int i = 0; i < valueLength; i++) {
+            value[i] = (byte) (random.nextInt(126 - 32) + 32);
+        }
+        BytesReference keyValue = new BytesArray(value);
+        return irc.new Key(entity, keyValue, UUID.randomUUID().toString()); // same UUID source as used in real key
+    }
+}
+

From 8cb83d7505d3a410391e62be85d1da9893556d02 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 13 Nov 2023 17:01:48 -0800
Subject: [PATCH 059/120] added null check to BR deserializer

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../opensearch/common/cache/tier/BytesReferenceSerializer.java | 3 +++
 .../common/cache/tier/keystore/RBMIntKeyLookupStore.java       | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java b/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
index 55ffe22c2a339..a2fa8fab9ea7f 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
@@ -25,6 +25,9 @@ public byte[] serialize(BytesReference object) {
 
     @Override
     public BytesReference deserialize(byte[] bytes) {
+        if (bytes == null) {
+            return null;
+        }
         return new BytesArray(bytes);
     }
 
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index e7370558da80b..bd77744196906 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -42,7 +42,7 @@
 import org.roaringbitmap.RoaringBitmap;
 
 /**
- * This class implements KeyLookupStore<Integer> using a roaring bitmap with a modulo applied to values.
+ * This class implements KeyLookupStore using a roaring bitmap with a modulo applied to values.
  * The modulo increases the density of values, which makes RBMs more memory-efficient. The recommended modulo is ~2^28.
  * It also maintains a hash set of values which have had collisions. Values which haven't had collisions can be
  * safely removed from the store. The fraction of collided values should be low,

From 111c4e663a17d96e07c7627c6b9b0b20e9cee743 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 14 Nov 2023 12:34:45 -0800
Subject: [PATCH 060/120] Fixed ehcache init issue in test cases

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../indices/IndicesRequestCacheDiskTierIT.java         |  1 +
 .../org/opensearch/indices/IndicesRequestCacheIT.java  |  1 +
 .../common/cache/tier/EhCacheDiskCachingTier.java      | 10 ++++++----
 .../index/cache/request/ShardRequestCache.java         |  2 +-
 .../org/opensearch/indices/IndicesRequestCache.java    |  8 +++++---
 .../java/org/opensearch/indices/IndicesService.java    |  4 ++--
 .../resources/org/opensearch/bootstrap/security.policy | 10 ++--------
 .../opensearch/indices/IndicesRequestCacheTests.java   |  2 +-
 8 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index e25fb0112ca01..b1db9b8e624aa 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.index.cache.request.RequestCacheStats;
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index 080b3017c4246..d5f24dc1babeb 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -40,6 +40,7 @@
 import org.opensearch.action.search.SearchType;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.common.util.FeatureFlags;
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index 895b3594077b9..bd0bd0a744b8e 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -57,7 +57,7 @@
 public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
 
     // A Cache manager can create many caches.
-    private final PersistentCacheManager cacheManager;
+    private static PersistentCacheManager cacheManager = null;
 
     // Disk cache
     private Cache<K, byte[]> cache;
@@ -126,8 +126,10 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.DISK_WRITE_CONCURRENCY = Setting.intSetting(builder.settingPrefix + ".tiered.disk.ehcache.concurrency", 2, 1, 3);
         // Default value is 16 within Ehcache.
         this.DISK_SEGMENTS = Setting.intSetting(builder.settingPrefix + ".ehcache.disk.segments", 16, 1, 32);
-        this.cacheManager = buildCacheManager();
-        this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
+        if (cacheManager == null) {
+            cacheManager = buildCacheManager();
+            this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
+        }
 
         // IndicesRequestCache gets 1%, of which we allocate 5% to the keystore = 0.05%
         // TODO: how do we change this automatically based on INDICES_CACHE_QUERY_SIZE setting?
@@ -150,7 +152,7 @@ private PersistentCacheManager buildCacheManager() {
     }
 
     private Cache<K, byte[]> buildCache(Duration expireAfterAccess, Builder<K, V> builder) {
-        return this.cacheManager.createCache(
+        return cacheManager.createCache(
             DISK_CACHE_ALIAS,
             CacheConfigurationBuilder.newCacheConfigurationBuilder(
                 keyType,
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index 6e7f4d335ecd1..385949d80af8b 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -86,7 +86,7 @@ public void onHit(CacheValue<BytesReference> cacheValue) {
 
     public void onMiss(CacheValue<BytesReference> cacheValue) {
         TierType source = cacheValue.getSource();
-        defaultStatsHolder.get(cacheValue.getSource()).missCount.inc();
+        defaultStatsHolder.get(source).missCount.inc();
         tierSpecificStatsHolder.get(source).addRequestStats(cacheValue.getStats());
     }
 
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 441229d9e3893..429a2452a9030 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -143,19 +143,21 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         // enabling this for testing purposes. Remove/tweak!!
         long CACHE_SIZE_IN_BYTES = 1000000L;
         String SETTING_PREFIX = "indices.request.cache";
+        String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
+        System.out.println("Node paths length = " + indicesService.getNodePaths().length);
 
-        /*EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
+        EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
             .setKeyType(Key.class)
             .setValueType(BytesReference.class)
             .setExpireAfterAccess(TimeValue.MAX_VALUE)
             .setSettings(settings)
             .setThreadPoolAlias("ehcacheTest")
             .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
-            .setStoragePath("/tmp")
+            .setStoragePath(STORAGE_PATH)
             .setSettingPrefix(SETTING_PREFIX)
             .setKeySerializer(new IRCKeyWriteableSerializer(this))
             .setValueSerializer(new BytesReferenceSerializer())
-            .build();*/
+            .build();
 
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
         tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference, QuerySearchResult>()
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index b5163d5ba34cb..6264dc7fec24d 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -1955,8 +1955,8 @@ public boolean allPendingDanglingIndicesWritten() {
             || (danglingIndicesToWrite.isEmpty() && danglingIndicesThreadPoolExecutor.getActiveCount() == 0);
     }
 
-    public String getNodeId() {
-        return nodeEnv.nodeId();
+    public NodeEnvironment.NodePath[] getNodePaths() {
+        return nodeEnv.nodePaths();
     }
 
     /**
diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy
index 1e588742f64c4..64decc7da12ca 100644
--- a/server/src/main/resources/org/opensearch/bootstrap/security.policy
+++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy
@@ -191,12 +191,6 @@ grant {
   // For ehcache
   permission java.lang.RuntimePermission "createClassLoader";
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
-
-  // Testing, remove later
-  //permission java.io.FilePermission "/tmp", "read";
-  //permission java.io.FilePermission "/tmp", "write";
-  //permission java.io.FilePermission "/tmp", "delete";
-  //permission java.io.FilePermission "/tmp/-", "read";
-  //permission java.io.FilePermission "/tmp/-", "write";
-  //permission java.io.FilePermission "/tmp/-", "delete";
+  permission java.lang.RuntimePermission "accessDeclaredMembers";
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 300d6f9ccc172..95c1056df5abe 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -269,7 +269,7 @@ public void testEviction() throws Exception {
         }
         IndicesRequestCache cache = new IndicesRequestCache(
             Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), size.getBytes() + 1 + "b").build(),
-            null,
+            getInstanceFromNode(IndicesService.class),
             dummyClusterSettings
         );
         AtomicBoolean indexShard = new AtomicBoolean(true);

From 55b09fe9dcd02e5089956961afcf68e6614f1305 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 14 Nov 2023 13:55:17 -0800
Subject: [PATCH 061/120] cleanup, fixed commented line

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../main/java/org/opensearch/indices/IndicesRequestCache.java  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 429a2452a9030..9ad08364665e9 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -144,7 +144,6 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         long CACHE_SIZE_IN_BYTES = 1000000L;
         String SETTING_PREFIX = "indices.request.cache";
         String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
-        System.out.println("Node paths length = " + indicesService.getNodePaths().length);
 
         EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
             .setKeyType(Key.class)
@@ -162,7 +161,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
         tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference, QuerySearchResult>()
             .setOnHeapCachingTier(openSearchOnHeapCache)
-            //.setOnDiskCachingTier(ehcacheDiskTier)
+            .setOnDiskCachingTier(ehcacheDiskTier)
             .setTieredCacheEventListener(this)
             .withPreDiskCachingPolicyFunction(transformationFunction)
             .withPolicy(new DiskTierTookTimePolicy(settings, clusterSettings))

From 24a3cb9e772a6533a375932fc76accd69328752b Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 14 Nov 2023 14:48:35 -0800
Subject: [PATCH 062/120] Fixed issues with closing disk tier in tests, added
 IT for disk specific stats

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../IndicesRequestCacheDiskTierIT.java        | 36 +++++++++++++++++++
 .../cache/tier/EhCacheDiskCachingTier.java    | 16 +++++----
 .../cache/request/RequestCacheStats.java      |  5 ++-
 .../cache/request/ShardRequestCache.java      |  4 +--
 .../indices/IndicesRequestCache.java          |  3 ++
 5 files changed, 54 insertions(+), 10 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index b1db9b8e624aa..83c28fb46a2c2 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -39,6 +39,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.index.cache.request.RequestCacheStats;
+import org.opensearch.index.cache.request.ShardRequestCache;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
@@ -84,10 +85,28 @@ public void testDiskTierStats() throws Exception {
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
         }
+
+        // So far, disk-specific stats should be 0, as keystore has prevented any actual disk reaches
+        long tookTimeSoFar = assertDiskTierSpecificStats(client, "index", 0, -1, 0);
+
         // the first request, for "hello0", should have been evicted to the disk tier
         resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello0")).get();
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 1, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 1, numRequests, TierType.DISK, false);
+        tookTimeSoFar = assertDiskTierSpecificStats(client, "index", 1, 0, -1);
+
+        // We make another actual request that should be in the disk tier. Disk specific stats should keep incrementing
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello1")).get();
+        IndicesRequestCacheIT.assertCacheState(client, "index", 0,  numRequests + 2, TierType.ON_HEAP, false);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 2, numRequests, TierType.DISK, false);
+        tookTimeSoFar = assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, -1);
+
+        // A final request for something in neither tier shouldn't increment disk specific stats
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + numRequests)).get();
+        IndicesRequestCacheIT.assertCacheState(client, "index", 0,  numRequests + 3, TierType.ON_HEAP, false);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 2, numRequests + 1, TierType.DISK, false);
+        assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, tookTimeSoFar);
+
     }
 
     private long getCacheSizeBytes(Client client, String index, TierType tierType) {
@@ -100,4 +119,21 @@ private long getCacheSizeBytes(Client client, String index, TierType tierType) {
             .getRequestCache();
         return requestCacheStats.getMemorySizeInBytes(tierType);
     }
+
+    private long assertDiskTierSpecificStats(Client client, String index, long totalDiskReaches, long totalGetTimeLowerBound, long totalGetTimeUpperBound) {
+        // set bounds to -1 to ignore them
+        RequestCacheStats requestCacheStats = client.admin()
+            .indices()
+            .prepareStats(index)
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+        ShardRequestCache.DiskStatsAccumulator specStats = requestCacheStats.getDiskSpecificStats();
+        assertEquals(totalDiskReaches, specStats.getTotalDiskReaches());
+        long tookTime = specStats.getTotalGetTime();
+        assertTrue(tookTime >= totalGetTimeLowerBound || totalGetTimeLowerBound < 0);
+        assertTrue(tookTime <= totalGetTimeUpperBound || totalGetTimeUpperBound < 0);
+        return tookTime; // Return for use in next check
+    }
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index bd0bd0a744b8e..dd3c17f11f095 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -126,10 +126,12 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.DISK_WRITE_CONCURRENCY = Setting.intSetting(builder.settingPrefix + ".tiered.disk.ehcache.concurrency", 2, 1, 3);
         // Default value is 16 within Ehcache.
         this.DISK_SEGMENTS = Setting.intSetting(builder.settingPrefix + ".ehcache.disk.segments", 16, 1, 32);
-        if (cacheManager == null) {
-            cacheManager = buildCacheManager();
-            this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
-        }
+
+        // In test cases, there might be leftover cache managers and caches hanging around, from nodes created in the test case setup
+        // Destroy them before recreating them
+        close();
+        cacheManager = buildCacheManager();
+        this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
 
         // IndicesRequestCache gets 1%, of which we allocate 5% to the keystore = 0.05%
         // TODO: how do we change this automatically based on INDICES_CACHE_QUERY_SIZE setting?
@@ -274,13 +276,13 @@ public TierType getTierType() {
 
     @Override
     public void close() {
-        cacheManager.removeCache(DISK_CACHE_ALIAS);
-        cacheManager.close();
         try {
             cacheManager.destroyCache(DISK_CACHE_ALIAS);
+            cacheManager.close();
+            cacheManager = null;
         } catch (CachePersistenceException e) {
             throw new OpenSearchException("Exception occurred while destroying ehcache and associated data", e);
-        }
+        } catch (NullPointerException ignored) {} // Another test node has already destroyed the cache manager
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index e3b1e7667d640..479c84d1b5254 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -65,7 +65,6 @@ public class RequestCacheStats implements Writeable, ToXContentFragment {
      private Map<String, ShardRequestCache.TierStatsAccumulator> tierSpecificStatsMap = new HashMap<>(){{
         put(TierType.ON_HEAP.getStringValue(), new ShardRequestCache.OnHeapStatsAccumulator());
         put(TierType.DISK.getStringValue(), new ShardRequestCache.DiskStatsAccumulator());
-        //checkTierSpecificMap(); // might yell, idk if the map is done until the block ends
      }};
 
     public RequestCacheStats() {}
@@ -115,6 +114,10 @@ ShardRequestCache.TierStatsAccumulator getTierSpecificStats(TierType tierType) {
         return tierSpecificStatsMap.get(tierType.getStringValue());
     }
 
+    public ShardRequestCache.DiskStatsAccumulator getDiskSpecificStats() {
+        return (ShardRequestCache.DiskStatsAccumulator) tierSpecificStatsMap.get(TierType.DISK.getStringValue());
+    }
+
     public long getMemorySizeInBytes(TierType tierType) {
         return getTierStats(tierType).totalMetric.count();
     }
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index 385949d80af8b..92c6846b7bbce 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -135,7 +135,7 @@ static abstract class TierStatsAccumulator<S extends TierRequestStats> implement
      * This class accumulates on-heap-tier-specific stats for a single shard.
      * For now, on-heap tier has no unique stats, but future stats would be added here.
      */
-    static class OnHeapStatsAccumulator extends TierStatsAccumulator<OnHeapTierRequestStats> {
+    public static class OnHeapStatsAccumulator extends TierStatsAccumulator<OnHeapTierRequestStats> {
         OnHeapStatsAccumulator() {}
         OnHeapStatsAccumulator(StreamInput in) {}
         @Override
@@ -156,7 +156,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * This class accumulates disk-tier-specific stats for a single shard.
      */
-    static class DiskStatsAccumulator extends TierStatsAccumulator<DiskTierRequestStats> {
+    public static class DiskStatsAccumulator extends TierStatsAccumulator<DiskTierRequestStats> {
         final CounterMetric totalGetTime;
         final CounterMetric totalDiskReaches; // Number of times a get() has actually reached the disk
         public DiskStatsAccumulator() {
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 9ad08364665e9..0b3c4ea404bc1 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -172,6 +172,9 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
     @Override
     public void close() {
         tieredCacheService.invalidateAll();
+        if (tieredCacheService.getDiskCachingTier().isPresent()) {
+            tieredCacheService.getDiskCachingTier().get().close();
+        }
     }
 
     void clear(CacheEntity entity) {

From 6a280c6d222f011b1378ac6b715eab06322cd148 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 14 Nov 2023 15:46:21 -0800
Subject: [PATCH 063/120] removed print statement

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../org/opensearch/indices/IndicesRequestCacheDiskTierIT.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index 83c28fb46a2c2..97f6cedb6add8 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -72,7 +72,6 @@ public void testDiskTierStats() throws Exception {
 
         resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
         int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
-        System.out.println(requestSize);
         assertTrue(heapSizeBytes > requestSize);
         // If this fails, increase heapSizeBytes! We can't adjust it after getting the size of one query
         // as the cache size setting is not dynamic

From 175c5c56a8fcd36d848562225f5d14ab7cbdc535 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 16 Nov 2023 13:21:59 -0800
Subject: [PATCH 064/120] Removed incorrect line from test

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../java/org/opensearch/indices/IndicesRequestCacheIT.java   | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index d5f24dc1babeb..965afd51d1356 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -676,9 +676,8 @@ public void testCacheWithInvalidation() throws Exception {
         // Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh)
         assertCacheState(client, "index", 1, 2, TierType.ON_HEAP, false);
         assertCacheState(client, "index", 0, 2, TierType.DISK, false);
-        assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP); // Shouldn't it just be the most recent query, since the first one was
-                                                                     // invalidated? (prob invalidation isnt in yet)
-        // yeah - evictions = 0, its not in yet
+
+        //assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP); // Evictions won't be 1 until the cache cleaner runs every minute
     }
 
     protected static void assertCacheState(

From af5178fca5e323b57f93fd9e902551a78bc7a6e9 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Fri, 17 Nov 2023 10:52:03 -0800
Subject: [PATCH 065/120] added required javadoc

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../common/cache/tier/BytesReferenceSerializer.java      | 4 ++++
 .../opensearch/common/cache/tier/CacheTierPolicy.java    | 3 +++
 .../common/cache/tier/EhCacheDiskCachingTier.java        | 1 +
 .../common/cache/tier/keystore/package-info.java         | 9 +++++++++
 4 files changed, 17 insertions(+)
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/keystore/package-info.java

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java b/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
index a2fa8fab9ea7f..201ce7e9c28a6 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
@@ -14,6 +14,10 @@
 import java.io.IOException;
 import java.util.Arrays;
 
+/**
+ * A serializer which transforms BytesReference to byte[].
+ * The type of BytesReference is NOT preserved after deserialization, but nothing in opensearch should care.
+ */
 public class BytesReferenceSerializer implements Serializer<BytesReference, byte[]> {
     // This class does not get passed to ehcache itself, so it's not required that classes match after deserialization.
 
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
index 8af9326d4a033..5f682995ab6d0 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
@@ -36,6 +36,9 @@
 
 import java.io.IOException;
 
+/**
+ * An interface for policies that inspect data of type T to decide whether they are admitted into a cache tier.
+ */
 public interface CacheTierPolicy<T> {
     /**
      * Determines whether this policy allows the data into its cache tier.
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index dd3c17f11f095..e0b59385de18d 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -51,6 +51,7 @@
 import org.opensearch.search.query.QuerySearchResult;
 
 /**
+ * An ehcache-based disk tier implementation.
  * @param <K> The key type of cache entries
  * @param <V> The value type of cache entries
  */
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/package-info.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/package-info.java
new file mode 100644
index 0000000000000..f3be16dfa3d32
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/package-info.java
@@ -0,0 +1,9 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+/** Package for disk tier keystore. */
+package org.opensearch.common.cache.tier.keystore;

From 7d1ce124cc0ab3cde6e9bd5b5360e48a2964c6a1 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 22 Nov 2023 11:21:33 -0800
Subject: [PATCH 066/120] Optimized modulo calculation with bitmask

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../cache/tier/keystore/RBMIntKeyLookupStore.java     | 11 +++++++++--
 .../tier/keystore/RBMIntKeyLookupStoreTests.java      | 10 +++++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index e7370558da80b..4e56e36c130c0 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -72,6 +72,8 @@ public int getValue() {
     }
 
     protected final int modulo;
+    protected final int modulo_bitmask;
+    // Since our modulo is always a power of two we can optimize it by ANDing with a particular bitmask
     KeyStoreStats stats;
     protected RoaringBitmap rbm;
     private HashMap<Integer, CounterMetric> collidedIntCounters;
@@ -92,6 +94,11 @@ public RBMIntKeyLookupStore(long memSizeCapInBytes) {
 
     public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBytes) {
         this.modulo = moduloValue.getValue();
+        if (modulo > 0) {
+            this.modulo_bitmask = modulo - 1; // keep last log_2(modulo) bits
+        } else {
+            this.modulo_bitmask = -1; // -1 in twos complement is all ones -> includes all bits -> same as no modulo
+        }
         this.stats = new KeyStoreStats(memSizeCapInBytes);
         this.rbm = new RoaringBitmap();
         this.collidedIntCounters = new HashMap<>();
@@ -99,8 +106,8 @@ public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBy
         this.mostRecentByteEstimate = 0L;
     }
 
-    private final int transform(int value) {
-        return modulo == 0 ? value : value % modulo;
+    private int transform(int value) {
+        return value & modulo_bitmask;
     }
 
     private void handleCollisions(int transformedValue) {
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index a6d076c8250ad..c0d16dfbc5744 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -61,14 +61,22 @@ public void testTransformationLogic() throws Exception {
         for (int i = 0; i < 4; i++) { // after this we run into max value, but thats not a flaw with the class design
             int posValue = i * modulo + offset;
             kls.add(posValue);
+            assertEquals(offset, (int) kls.getInternalRepresentation(posValue));
             int negValue = -(i * modulo + offset);
             kls.add(negValue);
+            assertEquals(modulo - offset, (int) kls.getInternalRepresentation(negValue));
         }
         assertEquals(2, kls.getSize());
         int[] testVals = new int[] { 0, 1, -1, -23495, 23058, modulo, -modulo, Integer.MAX_VALUE, Integer.MIN_VALUE };
         for (int value : testVals) {
             assertTrue(kls.getInternalRepresentation(value) < modulo);
-            assertTrue(kls.getInternalRepresentation(value) > -modulo);
+            assertTrue(kls.getInternalRepresentation(value) >= 0);
+        }
+        RBMIntKeyLookupStore no_modulo_kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L);
+        Random rand = Randomness.get();
+        for (int i = 0; i < 100; i++) {
+            int val = rand.nextInt();
+            assertEquals(val, (int) no_modulo_kls.getInternalRepresentation(val));
         }
     }
 

From 57e948fa5b5655b2baff426cdde605eafe26e126 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 22 Nov 2023 13:31:12 -0800
Subject: [PATCH 067/120] Addressed Sagar's comments on caching policy/generics

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../IndicesRequestCacheDiskTierIT.java        |  9 ++-
 .../cache/tier/CachePolicyInfoWrapper.java    | 46 +++++++++++++
 .../common/cache/tier/CacheTierPolicy.java    | 15 -----
 .../cache/tier/DiskTierTookTimePolicy.java    | 65 ++++++------------
 .../common/cache/tier/TieredCacheService.java | 12 +---
 .../TieredCacheSpilloverStrategyService.java  | 51 ++++----------
 .../indices/IndicesRequestCache.java          | 18 +++--
 .../opensearch/indices/IndicesService.java    |  7 ++
 .../tier/DiskTierTookTimePolicyTests.java     | 39 +++++++++--
 ...redCacheSpilloverStrategyServiceTests.java | 66 +++++--------------
 10 files changed, 153 insertions(+), 175 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index 97f6cedb6add8..4cb6eb1e3c09f 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -35,8 +35,10 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
 import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.cache.request.ShardRequestCache;
@@ -52,9 +54,11 @@
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class IndicesRequestCacheDiskTierIT extends OpenSearchIntegTestCase {
     public void testDiskTierStats() throws Exception {
-        int heapSizeBytes = 4729;
+        int heapSizeBytes = 9876;
         String node = internalCluster().startNode(
-            Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+                .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
         );
         Client client = client(node);
 
@@ -78,6 +82,7 @@ public void testDiskTierStats() throws Exception {
 
         int numOnDisk = 5;
         int numRequests = heapSizeBytes / requestSize + numOnDisk;
+        System.out.println("Size " + requestSize + " numRequests " + numRequests);
         for (int i = 1; i < numRequests; i++) {
             resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
             assertSearchResponse(resp);
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java b/server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java
new file mode 100644
index 0000000000000..6a077715e8232
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java
@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.common.cache.tier;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+
+import java.io.IOException;
+
+/**
+ * A class containing information needed for all CacheTierPolicy objects to decide whether to admit
+ * a given BytesReference. This spares us from having to create an entire short-lived QuerySearchResult object
+ * just to read a few values.
+ */
+public class CachePolicyInfoWrapper implements Writeable {
+    private final Long tookTimeNanos;
+    public CachePolicyInfoWrapper(Long tookTimeNanos) {
+        this.tookTimeNanos = tookTimeNanos;
+        // Add more values here as they are needed for future cache tier policies
+    }
+
+    public CachePolicyInfoWrapper(StreamInput in) throws IOException {
+        this.tookTimeNanos = in.readOptionalLong();
+    }
+
+    public Long getTookTimeNanos() {
+        return tookTimeNanos;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalLong(tookTimeNanos);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
index 5f682995ab6d0..dc884721d8e14 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
@@ -46,19 +46,4 @@ public interface CacheTierPolicy<T> {
      * @return true if accepted, otherwise false
      */
     boolean checkData(T data);
-
-    /**
-     * Checks whether the policy is currently being enforced.
-     */
-    boolean isActive();
-
-    /**
-     * Starts enforcing the policy if it is not already being enforced.
-     */
-    void activate();
-
-    /**
-     * Stops enforcing the policy. The policy will allow all input.
-     */
-    void deactivate();
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
index 417a40f002e6a..7de6583c485ed 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
@@ -6,25 +6,6 @@
  * compatible open source license.
  */
 
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
 /*
  * Modifications Copyright OpenSearch Contributors. See
  * GitHub history for details.
@@ -40,63 +21,55 @@
 import org.opensearch.search.query.QuerySearchResult;
 
 import java.io.IOException;
+import java.util.function.Function;
 
 /**
  * A cache tier policy which accepts queries whose took time is greater than some threshold,
  * which is specified as a dynamic cluster-level setting. The threshold should be set to approximately
  * the time it takes to get a result from the cache tier.
+ * The policy expects to be able to read a CachePolicyInfoWrapper from the start of the BytesReference.
  */
-public class DiskTierTookTimePolicy implements CacheTierPolicy<QuerySearchResult> {
+public class DiskTierTookTimePolicy implements CacheTierPolicy<BytesReference> {
     public static final Setting<TimeValue> DISK_TOOKTIME_THRESHOLD_SETTING = Setting.positiveTimeSetting(
         "index.requests.cache.disk.tooktime.threshold",
         new TimeValue(10),
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
-    );
+    ); // Set this to TimeValue.ZERO to let all data through
 
     private TimeValue threshold;
-    boolean isActive;
+    private final Function<BytesReference, CachePolicyInfoWrapper> getPolicyInfoFn;
 
-    public DiskTierTookTimePolicy(Settings settings, ClusterSettings clusterSettings) {
+    public DiskTierTookTimePolicy(Settings settings, ClusterSettings clusterSettings, Function<BytesReference, CachePolicyInfoWrapper> getPolicyInfoFn) {
         this.threshold = DISK_TOOKTIME_THRESHOLD_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(DISK_TOOKTIME_THRESHOLD_SETTING, this::setThreshold);
-        isActive = true;
+        this.getPolicyInfoFn = getPolicyInfoFn;
     }
 
-    protected void setThreshold(TimeValue threshold) { // public so that we can manually set value in unit test
+    protected void setThreshold(TimeValue threshold) { // protected so that we can manually set value in unit test
         this.threshold = threshold;
     }
 
     @Override
-    public boolean checkData(QuerySearchResult qsr) {
-        if (!isActive) {
+    public boolean checkData(BytesReference data) {
+        if (threshold.equals(TimeValue.ZERO)) {
             return true;
         }
-        Long tookTimeNanos = qsr.getTookTimeNanos();
-        if (tookTimeNanos == null) {
+        Long tookTimeNanos;
+        try {
+            tookTimeNanos = getPolicyInfoFn.apply(data).getTookTimeNanos();
+        } catch (Exception e) {
+            // If we can't retrieve the took time for whatever reason, admit the data to be safe
             return true;
+        }
+        if (tookTimeNanos == null) {
             // Received a null took time -> this QSR is from an old version which does not have took time, we should accept it
+            return true;
         }
-        TimeValue tookTime = TimeValue.timeValueNanos(qsr.getTookTimeNanos());
+        TimeValue tookTime = TimeValue.timeValueNanos(tookTimeNanos);
         if (tookTime.compareTo(threshold) < 0) { // negative -> tookTime is shorter than threshold
             return false;
         }
         return true;
     }
-
-    @Override
-    public boolean isActive() {
-        return isActive;
-    }
-
-    @Override
-    public void activate() {
-        isActive = true;
-
-    }
-
-    @Override
-    public void deactivate() {
-        isActive = false;
-    }
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
index 24fb9774fcb05..31d58510206f0 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
@@ -15,11 +15,8 @@
  * flavors like spillover etc.
  * @param <K> Type of key
  * @param <V> Type of value
- * @param <W> Type that V can be transformed into to inspect its contents to apply policies.
- *           For example, if V is BytesReference, W might be QuerySearchResult.
- *           Can be the same as V if no transformation is necessary
  */
-public interface TieredCacheService<K, V, W> {
+public interface TieredCacheService<K, V> {
 
     V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception;
 
@@ -34,11 +31,4 @@ public interface TieredCacheService<K, V, W> {
     OnHeapCachingTier<K, V> getOnHeapCachingTier();
 
     Optional<DiskCachingTier<K, V>> getDiskCachingTier();
-
-    /**
-     * Function transforming values of type V to type W so they can be inspected for contents before applying disk-tier policies.
-     * @param value
-     * @return
-     */
-    W preDiskCachingPolicyFunction(V value);
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 8aa56dc5bf156..78e4cc5e11f48 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -28,9 +28,8 @@
  * cache items to disk tier cache.
  * @param <K> Type of key
  * @param <V> Type of value
- * @param <W> Type that V can be unpacked into for inspection by policies. Can be the same as V.
  */
-public class TieredCacheSpilloverStrategyService<K, V, W> implements TieredCacheService<K, V, W>, RemovalListener<K, V> {
+public class TieredCacheSpilloverStrategyService<K, V> implements TieredCacheService<K, V>, RemovalListener<K, V> {
 
     private final OnHeapCachingTier<K, V> onHeapCachingTier;
 
@@ -44,17 +43,15 @@ public class TieredCacheSpilloverStrategyService<K, V, W> implements TieredCache
      * Maintains caching tiers in order of get calls.
      */
     private final List<CachingTier<K, V>> cachingTierList;
-    private final List<CacheTierPolicy<W>> policies;
-    private final Function<V, W> transformationFunction;
+    private final List<CacheTierPolicy<V>> policies;
 
-    private TieredCacheSpilloverStrategyService(Builder<K, V, W> builder) {
+    private TieredCacheSpilloverStrategyService(Builder<K, V> builder) {
         this.onHeapCachingTier = Objects.requireNonNull(builder.onHeapCachingTier);
         this.diskCachingTier = Optional.ofNullable(builder.diskCachingTier);
         this.tieredCacheEventListener = Objects.requireNonNull(builder.tieredCacheEventListener);
         this.cachingTierList = this.diskCachingTier.map(diskTier -> Arrays.asList(onHeapCachingTier, diskTier))
             .orElse(List.of(onHeapCachingTier));
         this.policies = Objects.requireNonNull(builder.policies);
-        this.transformationFunction = Objects.requireNonNull(builder.transformationFunction);
         setRemovalListeners();
     }
 
@@ -164,22 +161,9 @@ public Optional<DiskCachingTier<K, V>> getDiskCachingTier() {
         return this.diskCachingTier;
     }
 
-    @Override
-    public W preDiskCachingPolicyFunction(V value) {
-        return transformationFunction.apply(value);
-    }
-
     boolean checkPolicies(V value) {
-        W unpacked;
-        try {
-            unpacked = preDiskCachingPolicyFunction(value);
-        } catch (Exception e) {
-            // Bandaid solution: Accept any values that can't be unpacked into type W.
-            // TODO: Decide if this is reasonable, or if we should require the IRC to only accept BytesReferences which can be turned into QSRs.
-            return true;
-        }
-        for (CacheTierPolicy<W> policy : policies) {
-            if (!policy.checkData(unpacked)) {
+        for (CacheTierPolicy<V> policy : policies) {
+            if (!policy.checkData(value)) {
                 return false;
             }
         }
@@ -219,50 +203,43 @@ private Function<K, CacheValue<V>> getValueFromTierCache(boolean trackStats) {
      * Builder object
      * @param <K> Type of key
      * @param <V> Type of value
-     * @param <W> Type that V can be unpacked into for inspection by policies. Can be the same as V.
      */
-    public static class Builder<K, V, W> {
+    public static class Builder<K, V> {
         private OnHeapCachingTier<K, V> onHeapCachingTier;
         private DiskCachingTier<K, V> diskCachingTier;
         private TieredCacheEventListener<K, V> tieredCacheEventListener;
-        private ArrayList<CacheTierPolicy<W>> policies = new ArrayList<>();
-        private Function<V, W> transformationFunction;
+        private ArrayList<CacheTierPolicy<V>> policies = new ArrayList<>();
 
         public Builder() {}
 
-        public Builder<K, V, W> setOnHeapCachingTier(OnHeapCachingTier<K, V> onHeapCachingTier) {
+        public Builder<K, V> setOnHeapCachingTier(OnHeapCachingTier<K, V> onHeapCachingTier) {
             this.onHeapCachingTier = onHeapCachingTier;
             return this;
         }
 
-        public Builder<K, V, W> setOnDiskCachingTier(DiskCachingTier<K, V> diskCachingTier) {
+        public Builder<K, V> setOnDiskCachingTier(DiskCachingTier<K, V> diskCachingTier) {
             this.diskCachingTier = diskCachingTier;
             return this;
         }
 
-        public Builder<K, V, W> setTieredCacheEventListener(TieredCacheEventListener<K, V> tieredCacheEventListener) {
+        public Builder<K, V> setTieredCacheEventListener(TieredCacheEventListener<K, V> tieredCacheEventListener) {
             this.tieredCacheEventListener = tieredCacheEventListener;
             return this;
         }
 
-        public Builder<K, V, W> withPolicy(CacheTierPolicy<W> policy) {
+        public Builder<K, V> withPolicy(CacheTierPolicy<V> policy) {
             this.policies.add(policy);
             return this;
         }
 
         // Add multiple policies at once
-        public Builder<K, V, W> withPolicies(List<CacheTierPolicy<W>> policiesList) {
+        public Builder<K, V> withPolicies(List<CacheTierPolicy<V>> policiesList) {
             this.policies.addAll(policiesList);
             return this;
         }
 
-        public Builder<K, V, W> withPreDiskCachingPolicyFunction(Function<V, W> function) {
-            this.transformationFunction = function;
-            return this;
-        }
-
-        public TieredCacheSpilloverStrategyService<K, V, W> build() {
-            return new TieredCacheSpilloverStrategyService<K, V, W>(this);
+        public TieredCacheSpilloverStrategyService<K, V> build() {
+            return new TieredCacheSpilloverStrategyService<K, V>(this);
         }
     }
 
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 0b3c4ea404bc1..969974558c18e 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -41,6 +41,7 @@
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.tier.BytesReferenceSerializer;
+import org.opensearch.common.cache.tier.CachePolicyInfoWrapper;
 import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
 import org.opensearch.common.cache.tier.CacheValue;
 import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
@@ -120,7 +121,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
     private final Set<CleanupKey> keysToClean = ConcurrentCollections.newConcurrentSet();
     private final ByteSizeValue size;
     private final TimeValue expire;
-    private final TieredCacheService<Key, BytesReference, QuerySearchResult> tieredCacheService;
+    private final TieredCacheService<Key, BytesReference> tieredCacheService;
     private final IndicesService indicesService;
 
     IndicesRequestCache(Settings settings, IndicesService indicesService, ClusterSettings clusterSettings) {
@@ -133,9 +134,9 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             (k, v) -> k.ramBytesUsed() + v.ramBytesUsed()
         ).setMaximumWeight(sizeInBytes).setExpireAfterAccess(expire).build();
 
-        Function<BytesReference, QuerySearchResult> transformationFunction = (data) -> {
+        Function<BytesReference, CachePolicyInfoWrapper> transformationFunction = (data) -> {
             try {
-                return convertBytesReferenceToQSR(data);
+                return getPolicyInfo(data);
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
@@ -159,12 +160,11 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             .build();
 
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
-        tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference, QuerySearchResult>()
+        tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>()
             .setOnHeapCachingTier(openSearchOnHeapCache)
             .setOnDiskCachingTier(ehcacheDiskTier)
             .setTieredCacheEventListener(this)
-            .withPreDiskCachingPolicyFunction(transformationFunction)
-            .withPolicy(new DiskTierTookTimePolicy(settings, clusterSettings))
+            .withPolicy(new DiskTierTookTimePolicy(settings, clusterSettings, transformationFunction))
             .build();
         this.indicesService = indicesService;
     }
@@ -252,6 +252,12 @@ public static QuerySearchResult convertBytesReferenceToQSR(BytesReference data)
         return new QuerySearchResult(data.streamInput());
     }
 
+    public static CachePolicyInfoWrapper getPolicyInfo(BytesReference data) throws IOException {
+        // Reads the policy info corresponding to this QSR, written in IndicesService$loadIntoContext,
+        // without having to create a potentially large short-lived QSR object just for this purpose
+        return new CachePolicyInfoWrapper(data.streamInput());
+    }
+
     /**
      * Loader for the request cache
      *
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index 6264dc7fec24d..f513f067bd9be 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -61,6 +61,8 @@
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.cache.Cache;
+import org.opensearch.common.cache.tier.CachePolicyInfoWrapper;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
@@ -1690,6 +1692,10 @@ public void loadIntoContext(ShardSearchRequest request, SearchContext context, Q
         boolean[] loadedFromCache = new boolean[] { true };
         BytesReference bytesReference = cacheShardLevelResult(context.indexShard(), directoryReader, request.cacheKey(), out -> {
             queryPhase.execute(context);
+            CachePolicyInfoWrapper policyInfo = new CachePolicyInfoWrapper(context.queryResult().getTookTimeNanos());
+            policyInfo.writeTo(out);
+            // Write relevant info for cache tier policies before the whole QuerySearchResult, so we don't have to read
+            // the whole QSR into memory when we decide whether to allow it into a particular cache tier based on took time/other info
             context.queryResult().writeToNoId(out);
             loadedFromCache[0] = false;
         });
@@ -1698,6 +1704,7 @@ public void loadIntoContext(ShardSearchRequest request, SearchContext context, Q
             // restore the cached query result into the context
             final QuerySearchResult result = context.queryResult();
             StreamInput in = new NamedWriteableAwareStreamInput(bytesReference.streamInput(), namedWriteableRegistry);
+            CachePolicyInfoWrapper policyInfo = new CachePolicyInfoWrapper(in); // This wrapper is not needed outside the cache
             result.readFromWithId(context.id(), in);
             result.setSearchShardTarget(context.shardTarget());
         } else if (context.queryResult().searchTimedOut()) {
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java b/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
index b55aaf28cfd4a..9705a3fae4e24 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
@@ -45,9 +45,10 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
-import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.indices.IndicesRequestCache;
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.SearchShardTarget;
 import org.opensearch.search.internal.AliasFilter;
@@ -57,14 +58,21 @@
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.IOException;
-import java.util.Optional;
+import java.util.function.Function;
 
 public class DiskTierTookTimePolicyTests extends OpenSearchTestCase {
+    private final Function<BytesReference, CachePolicyInfoWrapper> transformationFunction = (data) -> {
+        try {
+            return IndicesRequestCache.getPolicyInfo(data);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    };
     private DiskTierTookTimePolicy getTookTimePolicy() {
         // dummy settings
         Settings dummySettings = Settings.EMPTY;
         ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        return new DiskTierTookTimePolicy(dummySettings, dummyClusterSettings);
+        return new DiskTierTookTimePolicy(dummySettings, dummyClusterSettings, transformationFunction);
     }
 
     public void testQSRSetupFunction() throws IOException {
@@ -80,12 +88,19 @@ public void testTookTimePolicy() throws Exception {
         long shortMillis = (long) (0.9 * threshMillis);
         long longMillis = (long) (1.5 * threshMillis);
         tookTimePolicy.setThreshold(new TimeValue((long) threshMillis));
-        QuerySearchResult shortQSR = getQSR(shortMillis * 1000000);
-        QuerySearchResult longQSR = getQSR(longMillis * 1000000);
+        BytesReference shortTime = getValidPolicyInput(getQSR(shortMillis * 1000000));
+        BytesReference longTime = getValidPolicyInput(getQSR(longMillis * 1000000));
 
-        boolean shortResult = tookTimePolicy.checkData(shortQSR);
+        boolean shortResult = tookTimePolicy.checkData(shortTime);
         assertFalse(shortResult);
-        boolean longResult = tookTimePolicy.checkData(longQSR);
+        boolean longResult = tookTimePolicy.checkData(longTime);
+        assertTrue(longResult);
+
+        DiskTierTookTimePolicy disabledPolicy = getTookTimePolicy();
+        disabledPolicy.setThreshold(TimeValue.ZERO);
+        shortResult = disabledPolicy.checkData(shortTime);
+        assertTrue(shortResult);
+        longResult = disabledPolicy.checkData(longTime);
         assertTrue(longResult);
     }
 
@@ -117,4 +132,14 @@ public static QuerySearchResult getQSR(long tookTimeNanos) {
         result.setTookTimeNanos(tookTimeNanos);
         return result;
     }
+
+    private BytesReference getValidPolicyInput(QuerySearchResult qsr) throws IOException {
+        // When it's used in the cache, the policy will receive BytesReferences which have a CachePolicyInfoWrapper
+        // at the beginning of them, followed by the actual QSR.
+        CachePolicyInfoWrapper policyInfo = new CachePolicyInfoWrapper(qsr.getTookTimeNanos());
+        BytesStreamOutput out = new BytesStreamOutput();
+        policyInfo.writeTo(out);
+        qsr.writeTo(out);
+        return out.bytes();
+    }
 }
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java b/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
index 533cc620aa600..f0b286b13b200 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
@@ -30,12 +30,11 @@ public class TieredCacheSpilloverStrategyServiceTests extends OpenSearchTestCase
     public void testComputeAndAbsentWithoutAnyOnHeapCacheEviction() throws Exception {
         int onHeapCacheSize = randomIntBetween(10, 30);
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
-        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = intializeTieredCacheService(
+        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = intializeTieredCacheService(
             onHeapCacheSize,
             randomIntBetween(1, 4),
             eventListener,
-                null,
-            null
+                null
         );
         int numOfItems1 = randomIntBetween(1, onHeapCacheSize / 2 - 1);
         List<String> keys = new ArrayList<>();
@@ -76,11 +75,10 @@ public void testComputeAndAbsentWithEvictionsFromOnHeapCache() throws Exception
         int diskCacheSize = randomIntBetween(60, 100);
         int totalSize = onHeapCacheSize + diskCacheSize;
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
-        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = intializeTieredCacheService(
+        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = intializeTieredCacheService(
             onHeapCacheSize,
             diskCacheSize,
             eventListener,
-            null,
             null
         );
 
@@ -145,11 +143,10 @@ public void testComputeAndAbsentWithEvictionsFromBothTier() throws Exception {
         int totalSize = onHeapCacheSize + diskCacheSize;
 
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
-        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = intializeTieredCacheService(
+        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = intializeTieredCacheService(
             onHeapCacheSize,
             diskCacheSize,
             eventListener,
-            null,
             null
         );
 
@@ -168,11 +165,10 @@ public void testGetAndCount() throws Exception {
         int totalSize = onHeapCacheSize + diskCacheSize;
 
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
-        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = intializeTieredCacheService(
+        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = intializeTieredCacheService(
             onHeapCacheSize,
             diskCacheSize,
             eventListener,
-            null,
             null
         );
 
@@ -211,12 +207,10 @@ public void testWithDiskTierNull() throws Exception {
         int onHeapCacheSize = randomIntBetween(10, 30);
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
         Function<String, String> identityFunction = (String value) -> { return value; };
-        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = new TieredCacheSpilloverStrategyService.Builder<
-            String,
+        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = new TieredCacheSpilloverStrategyService.Builder<
             String,
             String>().setOnHeapCachingTier(new MockOnHeapCacheTier<>(onHeapCacheSize))
             .setTieredCacheEventListener(eventListener)
-            .withPreDiskCachingPolicyFunction(identityFunction)
             .build();
         int numOfItems = randomIntBetween(onHeapCacheSize + 1, onHeapCacheSize * 3);
         for (int iter = 0; iter < numOfItems; iter++) {
@@ -229,12 +223,8 @@ public void testWithDiskTierNull() throws Exception {
         assertEquals(0, eventListener.enumMap.get(TierType.DISK).missCount.count());
     }
 
-    public void testTransformationFunctionAndDiskTierPolicy() throws Exception {
-        // For transformation function, return the string reversed
-        // For policy function, allow if what it receives starts with "a" (so original string ends with "a") and string is even length
-        Function<String, String> reverseStringFunction = (String value) -> {
-            return new StringBuilder(value).reverse().toString();
-        };
+    public void testDiskTierPolicies() throws Exception {
+        // For policy function, allow if what it receives starts with "a" and string is even length
         ArrayList<CacheTierPolicy<String>> policies = new ArrayList<>();
         policies.add(new AllowFirstLetterA());
         policies.add(new AllowEvenLengths());
@@ -242,24 +232,23 @@ public void testTransformationFunctionAndDiskTierPolicy() throws Exception {
         int onHeapCacheSize = 0;
         int diskCacheSize = 10000;
         MockTieredCacheEventListener<String, String> eventListener = new MockTieredCacheEventListener<String, String>();
-        TieredCacheSpilloverStrategyService<String, String, String> spilloverStrategyService = intializeTieredCacheService(
+        TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = intializeTieredCacheService(
             onHeapCacheSize,
             diskCacheSize,
             eventListener,
-            reverseStringFunction,
             policies
         );
 
         Map<String, String> keyValuePairs = new HashMap<>();
         Map<String, Boolean> expectedOutputs = new HashMap<>();
-        keyValuePairs.put("key1", "dcba");
+        keyValuePairs.put("key1", "abcd");
         expectedOutputs.put("key1", true);
-        keyValuePairs.put("key2", "edcba");
+        keyValuePairs.put("key2", "abcde");
         expectedOutputs.put("key2", false);
-        keyValuePairs.put("key3", "abc");
+        keyValuePairs.put("key3", "bbc");
         expectedOutputs.put("key3", false);
         keyValuePairs.put("key4", "ab");
-        expectedOutputs.put("key4", false);
+        expectedOutputs.put("key4", true);
         keyValuePairs.put("key5", "");
         expectedOutputs.put("key5", false);
 
@@ -289,14 +278,6 @@ public boolean checkData(String data) {
                 return false;
             }
         }
-        @Override
-        public boolean isActive() {
-            return true;
-        }
-        @Override
-        public void activate() {}
-        @Override
-        public void deactivate() {}
     }
 
     private static class AllowEvenLengths implements CacheTierPolicy<String> {
@@ -304,14 +285,6 @@ private static class AllowEvenLengths implements CacheTierPolicy<String> {
         public boolean checkData(String data) {
             return data.length() % 2 == 0;
         }
-        @Override
-        public boolean isActive() {
-            return true;
-        }
-        @Override
-        public void activate() {}
-        @Override
-        public void deactivate() {}
     }
 
     private TieredCacheLoader<String, String> getTieredCacheLoader() {
@@ -347,33 +320,24 @@ public boolean isLoaded() {
         };
     }
 
-    private TieredCacheSpilloverStrategyService<String, String, String> intializeTieredCacheService(
+    private TieredCacheSpilloverStrategyService<String, String> intializeTieredCacheService(
         int onHeapCacheSize,
         int diskCacheSize,
         TieredCacheEventListener<String, String> cacheEventListener,
-        Function<String, String> transformationFunction, // If passed null, default to identity function
         List<CacheTierPolicy<String>> policies // If passed null, default to no policies (empty list)
     ) {
         DiskCachingTier<String, String> diskCache = new MockDiskCachingTier<>(diskCacheSize);
         OnHeapCachingTier<String, String> openSearchOnHeapCache = new MockOnHeapCacheTier<>(onHeapCacheSize);
 
-        Function<String, String> function;
-        if (transformationFunction != null) {
-            function = transformationFunction;
-        } else {
-            function = (String value) -> value;
-        }
-
         List<CacheTierPolicy<String>> policiesToUse = new ArrayList<>();
         if (policies != null) {
             policiesToUse = policies;
         }
 
 
-        return new TieredCacheSpilloverStrategyService.Builder<String, String, String>().setOnHeapCachingTier(openSearchOnHeapCache)
+        return new TieredCacheSpilloverStrategyService.Builder<String, String>().setOnHeapCachingTier(openSearchOnHeapCache)
             .setOnDiskCachingTier(diskCache)
             .setTieredCacheEventListener(cacheEventListener)
-            .withPreDiskCachingPolicyFunction(function)
             .withPolicies(policiesToUse)
             .build();
     }

From c51e0f3bc8d69e85e3c1053427e6adce0d9cf2bc Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 22 Nov 2023 14:44:24 -0800
Subject: [PATCH 068/120] Addressed Sagar's other comments

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../common/cache/tier/EhCacheDiskCachingTier.java  | 14 +++++++++++---
 .../common/cache/tier/OnHeapTierRequestStats.java  |  2 +-
 .../opensearch/indices/IndicesRequestCache.java    |  4 ++++
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index e0b59385de18d..44e56072c4941 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -136,8 +136,10 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
 
         // IndicesRequestCache gets 1%, of which we allocate 5% to the keystore = 0.05%
         // TODO: how do we change this automatically based on INDICES_CACHE_QUERY_SIZE setting?
-        Setting<ByteSizeValue> keystoreSizeSetting = Setting.memorySizeSetting(builder.settingPrefix + ".tiered.disk.keystore_size", "0.05%");
-        this.keystore = new RBMIntKeyLookupStore(keystoreSizeSetting.get(this.settings).getBytes());
+        //Setting<ByteSizeValue> keystoreSizeSetting = Setting.memorySizeSetting(builder.settingPrefix + ".tiered.disk.keystore_size", "0.05%");
+        //this.keystore = new RBMIntKeyLookupStore(keystoreSizeSetting.get(this.settings).getBytes());
+        long keystoreMaxWeight = builder.keystoreMaxWeightInBytes;
+        this.keystore = new RBMIntKeyLookupStore(keystoreMaxWeight);
     }
 
     private PersistentCacheManager buildCacheManager() {
@@ -208,7 +210,7 @@ private CacheEventListenerConfigurationBuilder getListenerConfiguration(Builder<
     @Override
     public CacheValue<V> get(K key) {
         boolean reachedDisk = false;
-        long now = System.nanoTime(); // Nanoseconds required; milliseconds might be too slow on an SSD
+        long now = System.nanoTime();
 
         V value = null;
         if (keystore.contains(key.hashCode()) || keystore.isFull()) { // Check in-memory store of key hashes to avoid unnecessary disk seek
@@ -441,6 +443,7 @@ public static class Builder<K, V> {
         private boolean isEventListenerModeSync;
         private Serializer<K, byte[]> keySerializer;
         private Serializer<V, byte[]> valueSerializer;
+        private long keystoreMaxWeightInBytes = 0;
         public Builder() {}
 
         public EhCacheDiskCachingTier.Builder<K, V> setMaximumWeightInBytes(long sizeInBytes) {
@@ -507,6 +510,11 @@ public EhCacheDiskCachingTier.Builder<K, V> setValueSerializer(Serializer<V, byt
             return this;
         }
 
+        public EhCacheDiskCachingTier.Builder<K, V> setKeyStoreMaxWeightInBytes(long weight) {
+            this.keystoreMaxWeightInBytes = weight;
+            return this;
+        }
+
         public EhCacheDiskCachingTier<K, V> build() {
             return new EhCacheDiskCachingTier<>(this);
         }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java b/server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java
index 8f37f9e8a3141..3527dae885fde 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java
@@ -9,7 +9,7 @@
 package org.opensearch.common.cache.tier;
 
 /**
- * A class created by disk tier implementations containing disk-specific stats for a single request.
+ * A class created by on-heap tier implementations containing on-heap-specific stats for a single request.
  */
 public class OnHeapTierRequestStats implements TierRequestStats {
     @Override
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 969974558c18e..3ec20482fabf5 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -146,6 +146,9 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         String SETTING_PREFIX = "indices.request.cache";
         String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
 
+        double diskTierKeystoreWeightFraction = 0.05; // Allocate 5% of the on-heap weight to the disk tier's keystore
+        long keystoreMaxWeight = (long) (diskTierKeystoreWeightFraction * INDICES_CACHE_QUERY_SIZE.get(settings).getBytes());
+
         EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
             .setKeyType(Key.class)
             .setValueType(BytesReference.class)
@@ -157,6 +160,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             .setSettingPrefix(SETTING_PREFIX)
             .setKeySerializer(new IRCKeyWriteableSerializer(this))
             .setValueSerializer(new BytesReferenceSerializer())
+            .setKeyStoreMaxWeightInBytes(keystoreMaxWeight)
             .build();
 
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.

From 100ca7b5e01040cd3116fb6e78c0f202c250103d Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 22 Nov 2023 14:47:27 -0800
Subject: [PATCH 069/120] removed dead code

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../org/opensearch/indices/IndicesRequestCache.java    |  5 -----
 .../opensearch/indices/IndicesRequestCacheTests.java   | 10 ----------
 2 files changed, 15 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 3ec20482fabf5..2fbdd43abb23d 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -251,11 +251,6 @@ void invalidate(CacheEntity cacheEntity, DirectoryReader reader, BytesReference
         tieredCacheService.invalidate(new Key(cacheEntity, cacheKey, readerCacheKeyId));
     }
 
-    // Passed to TieredCacheService as its V -> W transformation function for inspecting BytesReferences in policies
-    public static QuerySearchResult convertBytesReferenceToQSR(BytesReference data) throws IOException, IllegalStateException {
-        return new QuerySearchResult(data.streamInput());
-    }
-
     public static CachePolicyInfoWrapper getPolicyInfo(BytesReference data) throws IOException {
         // Reads the policy info corresponding to this QSR, written in IndicesService$loadIntoContext,
         // without having to create a potentially large short-lived QSR object just for this purpose
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 95c1056df5abe..aee54dcf3379c 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -550,16 +550,6 @@ private static BytesReference getQSRBytesReference(long tookTimeNanos) throws IO
         return out.bytes();
     }
 
-    public void testBytesReferenceToQSRFunction() throws Exception {
-        BytesReference badQSR = new BytesArray("I love bytes!!!");
-        long ttn = 1000L;
-        BytesReference goodQSR = getQSRBytesReference(ttn);
-        //assertThrows(IOException.class, () -> IndicesRequestCache.convertBytesReferenceToQSR(badQSR));
-        // TODO: Uncomment this line if we decide to enforce IRC only accepting BytesReferences which can become QSRs.
-        QuerySearchResult unpackedQSR = IndicesRequestCache.convertBytesReferenceToQSR(goodQSR);
-        assertEquals(ttn, (long) unpackedQSR.getTookTimeNanos());
-    }
-
     private class TestBytesReference extends AbstractBytesReference {
 
         int dummyValue;

From c415ec86be00c3b2253a1d403ec774605edf98bf Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 22 Nov 2023 14:48:56 -0800
Subject: [PATCH 070/120] removed more dead code

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../common/cache/tier/EhCacheDiskCachingTier.java           | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index 44e56072c4941..f8aedbe5314de 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -133,11 +133,7 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         close();
         cacheManager = buildCacheManager();
         this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
-
-        // IndicesRequestCache gets 1%, of which we allocate 5% to the keystore = 0.05%
-        // TODO: how do we change this automatically based on INDICES_CACHE_QUERY_SIZE setting?
-        //Setting<ByteSizeValue> keystoreSizeSetting = Setting.memorySizeSetting(builder.settingPrefix + ".tiered.disk.keystore_size", "0.05%");
-        //this.keystore = new RBMIntKeyLookupStore(keystoreSizeSetting.get(this.settings).getBytes());
+        
         long keystoreMaxWeight = builder.keystoreMaxWeightInBytes;
         this.keystore = new RBMIntKeyLookupStore(keystoreMaxWeight);
     }

From 93fa4f8623534288dad400564a3e84bd13123bb1 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 27 Nov 2023 11:38:40 -0800
Subject: [PATCH 071/120] Changed memory size estimator to reflect optimized
 modulo

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../cache/tier/keystore/RBMIntKeyLookupStore.java     | 11 +++++++++--
 .../tier/keystore/RBMIntKeyLookupStoreTests.java      |  6 +++++-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index 4e56e36c130c0..e690deae7b521 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -82,7 +82,7 @@ public int getValue() {
     protected final Lock readLock = lock.readLock();
     protected final Lock writeLock = lock.writeLock();
     private long mostRecentByteEstimate;
-    private final int REFRESH_SIZE_EST_INTERVAL = 10000;
+    protected final int REFRESH_SIZE_EST_INTERVAL = 10000;
     // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms,
     // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store
     // in terms of storage impact
@@ -281,7 +281,14 @@ public boolean isCollision(Integer value1, Integer value2) {
     }
 
      static double getRBMSizeMultiplier(int numEntries, int modulo) {
-        double x = Math.log10((double) numEntries / modulo);
+        double effectiveModulo = (double) modulo / 2;
+        /* This model was created when we used % operator to calculate modulo. This has range (-modulo, modulo).
+        Now we have optimized to use a bitmask, which has range [0, modulo). So the number of possible values stored
+        is halved. */
+        if (modulo == 0) {
+            effectiveModulo = Math.pow(2, 32);
+        }
+        double x = Math.log10((double) numEntries / effectiveModulo);
         if (x < -5) {
             return 7.0;
         }
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index c0d16dfbc5744..d9b1ece1310ca 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -82,9 +82,13 @@ public void testTransformationLogic() throws Exception {
 
     public void testContains() throws Exception {
         RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
-        for (int i = 0; i < 2000; i++) {
+        RBMIntKeyLookupStore noModuloKls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L);
+        for (int i = 0; i < kls.REFRESH_SIZE_EST_INTERVAL + 1000; i++) {
+            // set upper bound > number of elements to trigger a size check, ensuring we test that too
             kls.add(i);
             assertTrue(kls.contains(i));
+            noModuloKls.add(i);
+            assertTrue(noModuloKls.contains(i));
         }
     }
 

From 52b9dfdb0b4d8b564eb7451e0344c363496aeab1 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Mon, 13 Nov 2023 17:24:07 -0800
Subject: [PATCH 072/120] Update IndicesRequestCache.java

to delete stale keys from diskcache
---
 .../indices/IndicesRequestCache.java          | 85 +++++++++++++++----
 1 file changed, 70 insertions(+), 15 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 2fbdd43abb23d..665c01b99d2c3 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -170,6 +170,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             .setTieredCacheEventListener(this)
             .withPolicy(new DiskTierTookTimePolicy(settings, clusterSettings, transformationFunction))
             .build();
+
         this.indicesService = indicesService;
     }
 
@@ -223,7 +224,7 @@ BytesReference getOrCompute(
         Loader cacheLoader = new Loader(cacheEntity, loader);
         BytesReference value = tieredCacheService.computeIfAbsent(key, cacheLoader);
         if (cacheLoader.isLoaded()) {
-            // see if its the first time we see this reader, and make sure to register a cleanup key
+            // see if it's the first time we see this reader, and make sure to register a cleanup key
             CleanupKey cleanupKey = new CleanupKey(cacheEntity, readerCacheKeyId);
             if (!registeredClosedListeners.containsKey(cleanupKey)) {
                 Boolean previous = registeredClosedListeners.putIfAbsent(cleanupKey, Boolean.TRUE);
@@ -425,36 +426,90 @@ public int hashCode() {
     /**
      * Logic to clean up in-memory cache.
      */
-    synchronized void cleanCache() {
+    synchronized void cleanCache() { // TODO rename this method to plural or cleanTieredCache ?
         final Set<CleanupKey> currentKeysToClean = new HashSet<>();
         final Set<Object> currentFullClean = new HashSet<>();
-        currentKeysToClean.clear();
-        currentFullClean.clear();
-        for (Iterator<CleanupKey> iterator = keysToClean.iterator(); iterator.hasNext();) {
+
+        Iterator<CleanupKey> iterator = keysToClean.iterator();
+        while(iterator.hasNext()) {
             CleanupKey cleanupKey = iterator.next();
             iterator.remove();
-            if (cleanupKey.readerCacheKeyId == null || cleanupKey.entity.isOpen() == false) {
-                // null indicates full cleanup, as does a closed shard
+            if (needsFullClean(cleanupKey)) {
                 currentFullClean.add(cleanupKey.entity.getCacheIdentity());
             } else {
                 currentKeysToClean.add(cleanupKey);
             }
         }
-        if (!currentKeysToClean.isEmpty() || !currentFullClean.isEmpty()) {
-            for (Iterator<Key> iterator = tieredCacheService.getOnHeapCachingTier().keys().iterator(); iterator.hasNext();) {
-                Key key = iterator.next();
-                if (currentFullClean.contains(key.entity.getCacheIdentity())) {
+
+        // Early exit if no cleanup is needed
+        if (currentKeysToClean.isEmpty() && currentFullClean.isEmpty()) {
+            return;
+        }
+        cleanTieredCaches(currentKeysToClean, currentFullClean);
+    }
+
+    private void cleanTieredCaches(Set<CleanupKey> currentKeysToClean, Set<Object> currentFullClean) {
+        cleanOnHeapCache(currentKeysToClean, currentFullClean);
+        cleanDiskCache(currentKeysToClean, currentFullClean);
+    }
+
+    // keeping this unsynchronized since we don't expect it to be called only by cleanCache which is synchronized
+    private void cleanDiskCache(Set<CleanupKey> currentKeysToClean, Set<Object> currentFullClean) {
+        if (tieredCacheService.getDiskCachingTier().isEmpty()) {
+            logger.debug("Skipping disk cache keys cleanup since no disk cache is configured");
+            return;
+        }
+        final double cleanupKeysThresholdPercentage = 50.0; // TODO make this an index setting
+        int totalKeysInDiskCache = tieredCacheService.getDiskCachingTier().get().count();
+        int totalKeysToCleanup = currentKeysToClean.size() + currentFullClean.size();
+
+        double cleanupKeysPercentage = ((double) totalKeysToCleanup / totalKeysInDiskCache) * 100;
+        if (cleanupKeysPercentage < cleanupKeysThresholdPercentage) {
+            logger.debug("Skipping disk cache keys cleanup since the keys to cleanup of {}% is not greater than " +
+                "the threshold percentage of {}%", cleanupKeysPercentage, cleanupKeysThresholdPercentage);
+            return;
+        }
+
+        Iterator<Key> iterator = tieredCacheService.getDiskCachingTier().get().keys().iterator();
+        while (iterator.hasNext()) {
+            Key key = iterator.next();
+            if (currentFullClean.contains(key.entity.getCacheIdentity())) {
+                iterator.remove();
+                currentFullClean.remove(key.entity.getCacheIdentity());
+            } else {
+                CleanupKey cleanupKey = new CleanupKey(key.entity, key.readerCacheKeyId);
+                if (currentKeysToClean.contains(cleanupKey)) {
+                    iterator.remove();
+                    currentKeysToClean.remove(cleanupKey);
+                }
+            }
+        }
+    }
+
+    // keeping this unsynchronized since we don't expect it to be called only by cleanCache which is synchronized
+    private void cleanOnHeapCache(Set<CleanupKey> currentKeysToClean, Set<Object> currentFullClean) {
+        Iterator<Key> iterator = tieredCacheService.getOnHeapCachingTier().keys().iterator();
+        while (iterator.hasNext()) {
+            Key key = iterator.next();
+            if (currentFullClean.contains(key.entity.getCacheIdentity())) {
+                iterator.remove();
+                currentFullClean.remove(key.entity.getCacheIdentity());
+            } else {
+                CleanupKey cleanupKey = new CleanupKey(key.entity, key.readerCacheKeyId);
+                if (currentKeysToClean.contains(cleanupKey)) {
                     iterator.remove();
-                } else {
-                    if (currentKeysToClean.contains(new CleanupKey(key.entity, key.readerCacheKeyId))) {
-                        iterator.remove();
-                    }
+                    currentKeysToClean.remove(cleanupKey);
                 }
             }
         }
         tieredCacheService.getOnHeapCachingTier().refresh();
     }
 
+    private boolean needsFullClean(CleanupKey cleanupKey) {
+        // null indicates full cleanup, as does a closed shard
+        return cleanupKey.readerCacheKeyId == null || !cleanupKey.entity.isOpen();
+    }
+
     /**
      * Returns the current size of the cache
      */

From 5bab65d449900c89b0a756c48a154d7a116129d3 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Thu, 16 Nov 2023 21:55:51 -0800
Subject: [PATCH 073/120] More updates to disk cache cleanup

---
 .../indices/IndicesRequestCache.java          | 138 ++++++++------
 .../opensearch/indices/IndicesService.java    | 136 +++++++++++---
 .../indices/IndicesRequestCacheTests.java     | 170 +++++++++++++++++-
 3 files changed, 360 insertions(+), 84 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 665c01b99d2c3..fb7662a57a6de 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -42,8 +42,9 @@
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.tier.BytesReferenceSerializer;
 import org.opensearch.common.cache.tier.CachePolicyInfoWrapper;
-import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
 import org.opensearch.common.cache.tier.CacheValue;
+import org.opensearch.common.cache.tier.CachingTier;
+import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
 import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
 import org.opensearch.common.cache.tier.OnHeapCachingTier;
 import org.opensearch.common.cache.tier.OpenSearchOnHeapCache;
@@ -64,7 +65,6 @@
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.common.unit.ByteSizeValue;
-import org.opensearch.search.query.QuerySearchResult;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -174,6 +174,19 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         this.indicesService = indicesService;
     }
 
+    // added for testing
+    IndicesRequestCache(
+        Settings settings,
+        IndicesService indicesService,
+        TieredCacheService<Key, BytesReference> tieredCacheService
+    ) {
+        this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
+        this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
+        long sizeInBytes = size.getBytes();
+        this.indicesService = indicesService;
+        this.tieredCacheService = tieredCacheService;
+    }
+
     @Override
     public void close() {
         tieredCacheService.invalidateAll();
@@ -238,9 +251,10 @@ BytesReference getOrCompute(
 
     /**
      * Invalidates the given the cache entry for the given key and it's context
+     *
      * @param cacheEntity the cache entity to invalidate for
-     * @param reader the reader to invalidate the cache entry for
-     * @param cacheKey the cache key to invalidate
+     * @param reader      the reader to invalidate the cache entry for
+     * @param cacheKey    the cache key to invalidate
      */
     void invalidate(CacheEntity cacheEntity, DirectoryReader reader, BytesReference cacheKey) {
         assert reader.getReaderCacheHelper() != null;
@@ -426,83 +440,87 @@ public int hashCode() {
     /**
      * Logic to clean up in-memory cache.
      */
-    synchronized void cleanCache() { // TODO rename this method to plural or cleanTieredCache ?
+    synchronized void cleanCache() {
         final Set<CleanupKey> currentKeysToClean = new HashSet<>();
         final Set<Object> currentFullClean = new HashSet<>();
 
-        Iterator<CleanupKey> iterator = keysToClean.iterator();
-        while(iterator.hasNext()) {
-            CleanupKey cleanupKey = iterator.next();
-            iterator.remove();
-            if (needsFullClean(cleanupKey)) {
-                currentFullClean.add(cleanupKey.entity.getCacheIdentity());
-            } else {
-                currentKeysToClean.add(cleanupKey);
-            }
-        }
+        categorizeKeysForCleanup(currentKeysToClean, currentFullClean);
 
         // Early exit if no cleanup is needed
         if (currentKeysToClean.isEmpty() && currentFullClean.isEmpty()) {
             return;
         }
-        cleanTieredCaches(currentKeysToClean, currentFullClean);
+
+        cleanUpKeys(
+            tieredCacheService.getOnHeapCachingTier().keys().iterator(),
+            currentKeysToClean,
+            currentFullClean
+        );
+        tieredCacheService.getOnHeapCachingTier().refresh();
     }
 
-    private void cleanTieredCaches(Set<CleanupKey> currentKeysToClean, Set<Object> currentFullClean) {
-        cleanOnHeapCache(currentKeysToClean, currentFullClean);
-        cleanDiskCache(currentKeysToClean, currentFullClean);
+    /**
+     * Logic to clean up disk based cache.
+     * <p>
+     * TODO: cleanDiskCache is very specific to disk caching tier. We can refactor this to be more generic.
+     */
+    synchronized void cleanDiskCache(double diskCachesCleanThresholdPercent) {
+        tieredCacheService.getDiskCachingTier().ifPresent(diskCachingTier -> {
+            if (diskCachingTier.count() == 0 || diskCleanupKeysPercentage() < diskCachesCleanThresholdPercent) {
+                if (logger.isDebugEnabled()) {
+                    logger.debug("Skipping disk cache keys cleanup");
+                    return;
+                }
+            }
+            Set<CleanupKey> currentKeysToClean = new HashSet<>();
+            Set<Object> currentFullClean = new HashSet<>();
+
+            categorizeKeysForCleanup(currentKeysToClean, currentFullClean);
+
+            // Early exit if no cleanup is needed
+            if (currentKeysToClean.isEmpty() && currentFullClean.isEmpty()) {
+                return;
+            }
+            cleanUpKeys(diskCachingTier.keys().iterator(), currentKeysToClean, currentFullClean);
+        });
     }
 
-    // keeping this unsynchronized since we don't expect it to be called only by cleanCache which is synchronized
-    private void cleanDiskCache(Set<CleanupKey> currentKeysToClean, Set<Object> currentFullClean) {
-        if (tieredCacheService.getDiskCachingTier().isEmpty()) {
-            logger.debug("Skipping disk cache keys cleanup since no disk cache is configured");
-            return;
-        }
-        final double cleanupKeysThresholdPercentage = 50.0; // TODO make this an index setting
-        int totalKeysInDiskCache = tieredCacheService.getDiskCachingTier().get().count();
-        int totalKeysToCleanup = currentKeysToClean.size() + currentFullClean.size();
-
-        double cleanupKeysPercentage = ((double) totalKeysToCleanup / totalKeysInDiskCache) * 100;
-        if (cleanupKeysPercentage < cleanupKeysThresholdPercentage) {
-            logger.debug("Skipping disk cache keys cleanup since the keys to cleanup of {}% is not greater than " +
-                "the threshold percentage of {}%", cleanupKeysPercentage, cleanupKeysThresholdPercentage);
-            return;
+    synchronized double diskCleanupKeysPercentage() {
+        int totalKeysInDiskCache = tieredCacheService.getDiskCachingTier()
+            .map(CachingTier::count)
+            .orElse(0);
+        if (totalKeysInDiskCache == 0 || keysToClean.isEmpty()) {
+            return 0;
         }
+        return ((double) keysToClean.size() / totalKeysInDiskCache) * 100;
+    }
 
-        Iterator<Key> iterator = tieredCacheService.getDiskCachingTier().get().keys().iterator();
+    synchronized void cleanUpKeys(Iterator<Key> iterator, Set<CleanupKey> currentKeysToClean, Set<Object> currentFullClean) {
         while (iterator.hasNext()) {
             Key key = iterator.next();
+            CleanupKey cleanupKey = new CleanupKey(key.entity, key.readerCacheKeyId);
             if (currentFullClean.contains(key.entity.getCacheIdentity())) {
                 iterator.remove();
                 currentFullClean.remove(key.entity.getCacheIdentity());
-            } else {
-                CleanupKey cleanupKey = new CleanupKey(key.entity, key.readerCacheKeyId);
-                if (currentKeysToClean.contains(cleanupKey)) {
-                    iterator.remove();
-                    currentKeysToClean.remove(cleanupKey);
-                }
+                keysToClean.remove(cleanupKey); // since a key could be either in onHeap or disk cache.
+            } else if (currentKeysToClean.contains(cleanupKey)) {
+                iterator.remove();
+                currentKeysToClean.remove(cleanupKey);
+                keysToClean.remove(cleanupKey);
             }
         }
     }
 
-    // keeping this unsynchronized since we don't expect it to be called only by cleanCache which is synchronized
-    private void cleanOnHeapCache(Set<CleanupKey> currentKeysToClean, Set<Object> currentFullClean) {
-        Iterator<Key> iterator = tieredCacheService.getOnHeapCachingTier().keys().iterator();
+    private void categorizeKeysForCleanup(Set<CleanupKey> currentKeysToClean, Set<Object> currentFullClean) {
+        Iterator<CleanupKey> iterator = keysToClean.iterator();
         while (iterator.hasNext()) {
-            Key key = iterator.next();
-            if (currentFullClean.contains(key.entity.getCacheIdentity())) {
-                iterator.remove();
-                currentFullClean.remove(key.entity.getCacheIdentity());
+            CleanupKey cleanupKey = iterator.next();
+            if (needsFullClean(cleanupKey)) {
+                currentFullClean.add(cleanupKey.entity.getCacheIdentity());
             } else {
-                CleanupKey cleanupKey = new CleanupKey(key.entity, key.readerCacheKeyId);
-                if (currentKeysToClean.contains(cleanupKey)) {
-                    iterator.remove();
-                    currentKeysToClean.remove(cleanupKey);
-                }
+                currentKeysToClean.add(cleanupKey);
             }
         }
-        tieredCacheService.getOnHeapCachingTier().refresh();
     }
 
     private boolean needsFullClean(CleanupKey cleanupKey) {
@@ -520,4 +538,16 @@ long count() {
     int numRegisteredCloseListeners() { // for testing
         return registeredClosedListeners.size();
     }
+
+    void addCleanupKeyForTesting(CacheEntity entity, String readerCacheKeyId) { // for testing
+        keysToClean.add(new CleanupKey(entity, readerCacheKeyId));
+    }
+
+    int getKeysToCleanSizeForTesting() { // for testing
+        return keysToClean.size();
+    }
+
+    Key createKeyForTesting(CacheEntity entity, String readerCacheKeyId) { // for testing
+        return new Key(entity, null, readerCacheKeyId);
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index f513f067bd9be..497458fd5927e 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -61,7 +61,6 @@
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.Nullable;
-import org.opensearch.common.cache.Cache;
 import org.opensearch.common.cache.tier.CachePolicyInfoWrapper;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lease.Releasable;
@@ -216,11 +215,24 @@ public class IndicesService extends AbstractLifecycleComponent
     private static final Logger logger = LogManager.getLogger(IndicesService.class);
 
     public static final String INDICES_SHARDS_CLOSED_TIMEOUT = "indices.shards_closed_timeout";
+    public static final String INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING_KEY = "cluster.request_cache.disk.cleanup_threshold_percentage";
+    public static final String INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING_DEFAULT_VALUE = "50%";
+
+    public static final String INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING_KEY = "indices.request_cache.disk.cleanup_interval";
+
+
     public static final Setting<TimeValue> INDICES_CACHE_CLEAN_INTERVAL_SETTING = Setting.positiveTimeSetting(
         "indices.cache.cleanup_interval",
         TimeValue.timeValueMinutes(1),
         Property.NodeScope
     );
+
+    public static final Setting<TimeValue> INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING = Setting.positiveTimeSetting(
+        INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING_KEY,
+        TimeValue.timeValueMinutes(1),
+        Property.NodeScope
+    );
+
     public static final Setting<Boolean> INDICES_ID_FIELD_DATA_ENABLED_SETTING = Setting.boolSetting(
         "indices.id_field_data.enabled",
         true,
@@ -228,6 +240,31 @@ public class IndicesService extends AbstractLifecycleComponent
         Property.NodeScope
     );
 
+    public static final Setting<String> INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING =
+        Setting.simpleString(
+            INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING_KEY,
+            INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING_DEFAULT_VALUE,
+            value -> {
+                String errorLogBase = "The value of the setting " +
+                    INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING_KEY +
+                    " must be ";
+                if (!value.endsWith("%")) {
+                    throw new IllegalArgumentException(errorLogBase + "a percentage");
+                }
+                String rawValue = value.substring(0, value.length() - 1);
+                try {
+                    double doubleValue = Double.parseDouble(rawValue);
+                    if (doubleValue < 0 || doubleValue > 100) {
+                        throw new IllegalArgumentException(errorLogBase + "between 0% and 100%");
+                    }
+                } catch (NumberFormatException e) {
+                    throw new IllegalArgumentException(errorLogBase + "a valid percentage", e);
+                }
+            },
+            Property.Dynamic,
+            Property.NodeScope
+        );
+
     public static final Setting<Boolean> WRITE_DANGLING_INDICES_INFO_SETTING = Setting.boolSetting(
         "gateway.write_dangling_indices_info",
         true,
@@ -325,6 +362,7 @@ public class IndicesService extends AbstractLifecycleComponent
     private final IndexScopedSettings indexScopedSettings;
     private final IndicesFieldDataCache indicesFieldDataCache;
     private final CacheCleaner cacheCleaner;
+    private final DiskCacheCleaner diskCacheCleaner;
     private final ThreadPool threadPool;
     private final CircuitBreakerService circuitBreakerService;
     private final BigArrays bigArrays;
@@ -338,7 +376,9 @@ public class IndicesService extends AbstractLifecycleComponent
     private final MapperRegistry mapperRegistry;
     private final NamedWriteableRegistry namedWriteableRegistry;
     private final IndexingMemoryController indexingMemoryController;
-    private final TimeValue cleanInterval;
+    private final TimeValue onHeapCachesCleanInterval;
+    private final TimeValue diskCachesCleanInterval;
+    private final double diskCachesCleanThreshold;
     final IndicesRequestCache indicesRequestCache; // pkg-private for testing
     private final IndicesQueryCache indicesQueryCache;
     private final MetaStateService metaStateService;
@@ -366,8 +406,9 @@ public class IndicesService extends AbstractLifecycleComponent
 
     @Override
     protected void doStart() {
-        // Start thread that will manage cleaning the field data cache periodically
-        threadPool.schedule(this.cacheCleaner, this.cleanInterval, ThreadPool.Names.SAME);
+        // Start threads that will manage cleaning the field data and request caches periodically
+        threadPool.schedule(this.cacheCleaner, this.onHeapCachesCleanInterval, ThreadPool.Names.SAME);
+        threadPool.schedule(this.diskCacheCleaner, this.diskCachesCleanInterval, ThreadPool.Names.SAME);
     }
 
     public IndicesService(
@@ -435,8 +476,11 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon
                 circuitBreakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(-sizeInBytes);
             }
         });
-        this.cleanInterval = INDICES_CACHE_CLEAN_INTERVAL_SETTING.get(settings);
-        this.cacheCleaner = new CacheCleaner(indicesFieldDataCache, indicesRequestCache, logger, threadPool, this.cleanInterval);
+        this.onHeapCachesCleanInterval = INDICES_CACHE_CLEAN_INTERVAL_SETTING.get(settings);
+        this.diskCachesCleanInterval = INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING.get(settings);
+        this.diskCachesCleanThreshold = getCleanupKeysThresholdPercentage(settings);;
+        this.cacheCleaner = new CacheCleaner(indicesFieldDataCache, indicesRequestCache, logger, threadPool, this.onHeapCachesCleanInterval);
+        this.diskCacheCleaner = new DiskCacheCleaner(indicesRequestCache, logger, threadPool, this.diskCachesCleanInterval, this.diskCachesCleanThreshold);
         this.metaStateService = metaStateService;
         this.engineFactoryProviders = engineFactoryProviders;
 
@@ -496,6 +540,12 @@ protected void closeInternal() {
         this.recoverySettings = recoverySettings;
     }
 
+    private double getCleanupKeysThresholdPercentage(Settings settings) {
+        String threshold = INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING.get(settings);
+        String rawValue = threshold.substring(0, threshold.length() - 1);
+        return Double.parseDouble(rawValue) / 100;
+    }
+
     /**
      * The changes to dynamic cluster setting {@code cluster.default.index.refresh_interval} needs to be updated. This
      * method gets called whenever the setting changes. We set the instance variable with the updated value as this is
@@ -1564,21 +1614,37 @@ public AnalysisRegistry getAnalysis() {
         return analysisRegistry;
     }
 
+    private static abstract class AbstractCacheCleaner implements Runnable, Releasable {
+
+        protected final Logger logger;
+        protected final ThreadPool threadPool;
+        protected final TimeValue interval;
+        protected final AtomicBoolean closed = new AtomicBoolean(false);
+
+        AbstractCacheCleaner(
+            Logger logger,
+            ThreadPool threadPool,
+            TimeValue interval
+        ) {
+            this.logger = logger;
+            this.threadPool = threadPool;
+            this.interval = interval;
+        }
+
+        @Override
+        public void close() {
+            closed.compareAndSet(false, true);
+        }
+    }
+
     /**
-     * FieldDataCacheCleaner is a scheduled Runnable used to clean a Guava cache
-     * periodically. In this case it is the field data cache, because a cache that
-     * has an entry invalidated may not clean up the entry if it is not read from
-     * or written to after invalidation.
+     * CacheCleaner is a scheduled Runnable used to clean Field Data Caches and/or request caches periodically.
      *
      * @opensearch.internal
      */
-    private static final class CacheCleaner implements Runnable, Releasable {
+    private static final class CacheCleaner extends AbstractCacheCleaner implements Runnable, Releasable {
 
         private final IndicesFieldDataCache cache;
-        private final Logger logger;
-        private final ThreadPool threadPool;
-        private final TimeValue interval;
-        private final AtomicBoolean closed = new AtomicBoolean(false);
         private final IndicesRequestCache requestCache;
 
         CacheCleaner(
@@ -1588,11 +1654,9 @@ private static final class CacheCleaner implements Runnable, Releasable {
             ThreadPool threadPool,
             TimeValue interval
         ) {
+            super(logger, threadPool, interval);
             this.cache = cache;
             this.requestCache = requestCache;
-            this.logger = logger;
-            this.threadPool = threadPool;
-            this.interval = interval;
         }
 
         @Override
@@ -1620,13 +1684,43 @@ public void run() {
             }
             // Reschedule itself to run again if not closed
             if (closed.get() == false) {
-                threadPool.scheduleUnlessShuttingDown(interval, ThreadPool.Names.SAME, this);
+                this.threadPool.scheduleUnlessShuttingDown(interval, ThreadPool.Names.SAME, this);
             }
         }
+    }
+
+    private static final class DiskCacheCleaner extends AbstractCacheCleaner implements Runnable, Releasable {
+
+        private final IndicesRequestCache requestCache;
+        private final double diskCachesCleanThresholdPercent;
+
+        DiskCacheCleaner(
+            IndicesRequestCache requestCache,
+            Logger logger,
+            ThreadPool threadPool,
+            TimeValue interval,
+            double diskCachesCleanThresholdPercent
+        ) {
+            super(logger, threadPool, interval);
+            this.diskCachesCleanThresholdPercent = diskCachesCleanThresholdPercent;
+            this.requestCache = requestCache;
+        }
 
         @Override
-        public void close() {
-            closed.compareAndSet(false, true);
+        public void run() {
+            long startTimeNS = System.nanoTime();
+            if (logger.isTraceEnabled()) {
+                logger.trace("running periodic disk based request cache cleanup");
+            }
+            try {
+                this.requestCache.cleanDiskCache(diskCachesCleanThresholdPercent);
+            } catch (Exception e) {
+                logger.warn("Exception during periodic request cache cleanup:", e);
+            }
+            // Reschedule itself to run again if not closed
+            if (closed.get() == false) {
+                threadPool.scheduleUnlessShuttingDown(interval, ThreadPool.Names.SAME, this);
+            }
         }
     }
 
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index aee54dcf3379c..b323cccc16290 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -52,6 +52,13 @@
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.UUIDs;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.opensearch.common.CheckedSupplier;
+import org.opensearch.common.cache.tier.DiskCachingTier;
+import org.opensearch.common.cache.tier.TieredCacheService;
+import org.opensearch.common.cache.tier.TieredCacheSpilloverStrategyService;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
 import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
@@ -78,13 +85,23 @@
 import org.opensearch.search.internal.ShardSearchContextId;
 import org.opensearch.search.internal.ShardSearchRequest;
 import org.opensearch.search.query.QuerySearchResult;
+import org.opensearch.indices.IndicesRequestCache.CacheEntity;
+import org.opensearch.indices.IndicesRequestCache.Key;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.internal.verification.VerificationModeFactory.atLeastOnce;
+
 public class IndicesRequestCacheTests extends OpenSearchSingleNodeTestCase {
 
     public void testBasicOperationsCache() throws Exception {
@@ -291,7 +308,7 @@ public void testEviction() throws Exception {
 
         writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
         DirectoryReader thirdReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TestEntity thirddEntity = new TestEntity(requestCacheStats, indexShard);
+        TestEntity thirdEntity = new TestEntity(requestCacheStats, indexShard);
         Loader thirdLoader = new Loader(thirdReader, 0);
 
         BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes);
@@ -299,7 +316,7 @@ public void testEviction() throws Exception {
         BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes);
         assertEquals("bar", value2.streamInput().readString());
         logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
-        BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes);
+        BytesReference value3 = cache.getOrCompute(thirdEntity, thirdLoader, thirdReader, termBytes);
         assertEquals("baz", value3.streamInput().readString());
         assertEquals(2, cache.count());
         assertEquals(1, requestCacheStats.stats().getEvictions());
@@ -330,7 +347,7 @@ public void testClearAllEntityIdentity() throws Exception {
         writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
         DirectoryReader thirdReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
         AtomicBoolean differentIdentity = new AtomicBoolean(true);
-        TestEntity thirddEntity = new TestEntity(requestCacheStats, differentIdentity);
+        TestEntity thirdEntity = new TestEntity(requestCacheStats, differentIdentity);
         Loader thirdLoader = new Loader(thirdReader, 0);
 
         BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes);
@@ -338,16 +355,16 @@ public void testClearAllEntityIdentity() throws Exception {
         BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes);
         assertEquals("bar", value2.streamInput().readString());
         logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
-        BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes);
+        BytesReference value3 = cache.getOrCompute(thirdEntity, thirdLoader, thirdReader, termBytes);
         assertEquals("baz", value3.streamInput().readString());
         assertEquals(3, cache.count());
         final long hitCount = requestCacheStats.stats().getHitCount();
-        // clear all for the indexShard Idendity even though is't still open
+        // clear all for the indexShard Identity even though it isn't still open
         cache.clear(randomFrom(entity, secondEntity));
         cache.cleanCache();
         assertEquals(1, cache.count());
         // third has not been validated since it's a different identity
-        value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes);
+        value3 = cache.getOrCompute(thirdEntity, thirdLoader, thirdReader, termBytes);
         assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount());
         assertEquals("baz", value3.streamInput().readString());
 
@@ -357,8 +374,8 @@ public void testClearAllEntityIdentity() throws Exception {
 
     public Iterable<Field> newDoc(int id, String value) {
         return Arrays.asList(
-            newField("id", Integer.toString(id), StringField.TYPE_STORED),
-            newField("value", value, StringField.TYPE_STORED)
+                newField("id", Integer.toString(id), StringField.TYPE_STORED),
+                newField("value", value, StringField.TYPE_STORED)
         );
     }
 
@@ -631,6 +648,141 @@ public long ramBytesUsed() {
         }
 
         @Override
-        public void writeTo(StreamOutput out) throws IOException {}
+        public void writeTo(StreamOutput out) throws IOException {
+        }
+    }
+
+    public static class CleanDiskCacheTests {
+        private IndicesRequestCache indicesRequestCache;
+        private TieredCacheService<Key, BytesReference> tieredCacheService;
+        private DiskCachingTier<Key, BytesReference> diskCachingTier;
+        private IndicesService indicesService;
+
+        @Before
+        public void setup() {
+            tieredCacheService = mock(TieredCacheService.class);
+            diskCachingTier = mock(DiskCachingTier.class);
+            indicesService = mock(IndicesService.class);
+            indicesRequestCache = new IndicesRequestCache(Settings.EMPTY, indicesService, tieredCacheService);
+        }
+
+        @Test
+        public void shouldNotCleanDiskCacheWhenEmpty() {
+            final int DISK_CACHE_COUNT = 0;
+            final double CLEANUP_THRESHOLD = 50.0;
+
+            when(tieredCacheService.getDiskCachingTier()).thenReturn(
+                    (Optional<DiskCachingTier<Key, BytesReference>>) Optional.of(diskCachingTier)
+            );
+            when(diskCachingTier.count()).thenReturn(DISK_CACHE_COUNT);
+
+            indicesRequestCache.cleanDiskCache(CLEANUP_THRESHOLD);
+
+            verify(diskCachingTier, never()).keys();
+        }
+
+        @Test
+        public void shouldNotCleanDiskCacheWhenCleanupKeysPercentageIsBelowThreshold() {
+            final int DISK_CACHE_COUNT = 1;
+            final double CLEANUP_THRESHOLD = 49.0;
+
+            when(tieredCacheService.getDiskCachingTier()).thenReturn(
+                    (Optional<DiskCachingTier<Key, BytesReference>>) Optional.of(diskCachingTier)
+            );
+            when(diskCachingTier.count()).thenReturn(DISK_CACHE_COUNT);
+
+            indicesRequestCache.cleanDiskCache(CLEANUP_THRESHOLD);
+
+            verify(diskCachingTier, never()).keys();
+        }
+
+        @Test
+        public void cleanDiskCacheWhenCleanupKeysPercentageIsGreaterThanOrEqualToThreshold() {
+            final int DISK_CACHE_COUNT = 100;
+            final double CLEANUP_THRESHOLD = 50.0;
+
+            // Mock dependencies
+            IndicesService mockIndicesService = mock(IndicesService.class);
+            TieredCacheService<Key, BytesReference> mockTieredCacheService = mock(TieredCacheService.class);
+            DiskCachingTier<Key, BytesReference> mockDiskCachingTier = mock(DiskCachingTier.class);
+            Iterator<Key> mockIterator = mock(Iterator.class);
+            Iterable<Key> mockIterable = () -> mockIterator;
+
+            IndicesRequestCache cache = new IndicesRequestCache(
+                    Settings.EMPTY,
+                    mockIndicesService,
+                    mockTieredCacheService
+            );
+
+            // Set up mocks
+            when(mockTieredCacheService.getDiskCachingTier()).thenReturn(Optional.of(mockDiskCachingTier));
+            when(mockDiskCachingTier.count()).thenReturn(DISK_CACHE_COUNT);
+            when(mockDiskCachingTier.keys()).thenReturn(mockIterable);
+            when(mockIterator.hasNext()).thenReturn(true, true, false);
+
+            // Create mock Keys and return them when next() is called
+            CacheEntity mockEntity = mock(CacheEntity.class);
+            Key firstMockKey = cache.createKeyForTesting(mockEntity, "readerCacheKeyId1");
+            Key secondMockKey = cache.createKeyForTesting(mockEntity, "readerCacheKeyId2");
+            when(mockEntity.getCacheIdentity()).thenReturn(new Object());
+            when(mockIterator.next()).thenReturn(firstMockKey, secondMockKey);
+
+            cache.addCleanupKeyForTesting(mockEntity, "readerCacheKeyId");
+            cache.cleanDiskCache(CLEANUP_THRESHOLD);
+
+            // Verify interactions
+            verify(mockDiskCachingTier).keys();
+            verify(mockIterator, atLeastOnce()).remove();
+        }
+
+        @Test
+        public void diskCleanupKeysPercentageWhenDiskCacheIsEmpty() {
+            when(tieredCacheService.getDiskCachingTier()).thenReturn(Optional.of(diskCachingTier));
+            when(diskCachingTier.count()).thenReturn(0);
+
+            double result = indicesRequestCache.diskCleanupKeysPercentage();
+
+            assertEquals(0, result, 0);
+        }
+
+        @Test
+        public void diskCleanupKeysPercentageWhenKeysToCleanIsEmpty() {
+            IndicesService mockIndicesService = mock(IndicesService.class);
+            TieredCacheService<Key, BytesReference> mockTieredCacheService = mock(TieredCacheSpilloverStrategyService.class);
+            DiskCachingTier<Key, BytesReference> mockDiskCachingTier = mock(DiskCachingTier.class);
+            when(mockTieredCacheService.getDiskCachingTier()).thenReturn(Optional.of(mockDiskCachingTier));
+            when(mockDiskCachingTier.count()).thenReturn(100);
+            IndicesRequestCache cache = new IndicesRequestCache(
+                    Settings.EMPTY,
+                    mockIndicesService,
+                    mockTieredCacheService
+            );
+
+            double result = cache.diskCleanupKeysPercentage();
+
+            assertEquals(0, result, 0.001);
+        }
+
+        @Test
+        public void diskCleanupKeysPercentageWhenDiskCacheAndKeysToCleanAreNotEmpty() {
+            IndicesService mockIndicesService = mock(IndicesService.class);
+            TieredCacheService<Key, BytesReference> mockTieredCacheService = mock(TieredCacheSpilloverStrategyService.class);
+            DiskCachingTier<Key, BytesReference> mockDiskCachingTier = mock(DiskCachingTier.class);
+            when(mockTieredCacheService.getDiskCachingTier()).thenReturn(Optional.of(mockDiskCachingTier));
+            when(mockDiskCachingTier.count()).thenReturn(100);
+
+            IndicesRequestCache cache = new IndicesRequestCache(
+                    Settings.EMPTY,
+                    mockIndicesService,
+                    mockTieredCacheService
+            );
+
+            IndicesRequestCache.CacheEntity mockEntity = Mockito.mock(IndicesRequestCache.CacheEntity.class);
+            when(mockEntity.getCacheIdentity()).thenReturn(new Object());
+            cache.addCleanupKeyForTesting(mockEntity, "readerCacheKeyId");
+
+            double result = cache.diskCleanupKeysPercentage();
+            assertEquals(1.0, result, 0);
+        }
     }
 }

From 4500090f5adbd834bb62cbd8e6df200c87b12a8d Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 28 Nov 2023 21:46:54 -0800
Subject: [PATCH 074/120] register
 INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING in clustersettings.java

---
 .../java/org/opensearch/common/settings/ClusterSettings.java | 3 ++-
 .../src/main/java/org/opensearch/indices/IndicesService.java | 5 ++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 5e3f4e959648e..0b0dfcc63c422 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -688,11 +688,12 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING,
                 IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
                 IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING,
+                IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING,
+                IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING,
                 AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT,
                 CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT,
-                IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING,
                 DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING
             )
         )
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index 497458fd5927e..c92b9170bec35 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -242,11 +242,10 @@ public class IndicesService extends AbstractLifecycleComponent
 
     public static final Setting<String> INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING =
         Setting.simpleString(
-            INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING_KEY,
+            "cluster.request_cache.disk.cleanup_threshold_percentage",
             INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING_DEFAULT_VALUE,
             value -> {
-                String errorLogBase = "The value of the setting " +
-                    INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING_KEY +
+                String errorLogBase = "Setting 'cluster.request_cache.disk.cleanup_threshold_percentage' " +
                     " must be ";
                 if (!value.endsWith("%")) {
                     throw new IllegalArgumentException(errorLogBase + "a percentage");

From 4199bc2726235456e5b5422eaf4e836f25c2c5ed Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 28 Nov 2023 21:48:34 -0800
Subject: [PATCH 075/120] make
 INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING dynamic property

---
 server/src/main/java/org/opensearch/indices/IndicesService.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index c92b9170bec35..253238c96e642 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -230,6 +230,7 @@ public class IndicesService extends AbstractLifecycleComponent
     public static final Setting<TimeValue> INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING = Setting.positiveTimeSetting(
         INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING_KEY,
         TimeValue.timeValueMinutes(1),
+        Property.Dynamic,
         Property.NodeScope
     );
 

From 6e05b9dce6a355ee8e571da34ccaf3ee6109577a Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 28 Nov 2023 21:49:44 -0800
Subject: [PATCH 076/120] minor changes

---
 .../indices/IndicesRequestCache.java           | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index fb7662a57a6de..9fb015e5ca695 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -452,7 +452,7 @@ synchronized void cleanCache() {
         }
 
         cleanUpKeys(
-            tieredCacheService.getOnHeapCachingTier().keys().iterator(),
+            tieredCacheService.getOnHeapCachingTier(),
             currentKeysToClean,
             currentFullClean
         );
@@ -481,7 +481,10 @@ synchronized void cleanDiskCache(double diskCachesCleanThresholdPercent) {
             if (currentKeysToClean.isEmpty() && currentFullClean.isEmpty()) {
                 return;
             }
-            cleanUpKeys(diskCachingTier.keys().iterator(), currentKeysToClean, currentFullClean);
+            cleanUpKeys(
+                tieredCacheService.getOnHeapCachingTier(),
+                currentKeysToClean,
+                currentFullClean);
         });
     }
 
@@ -495,16 +498,21 @@ synchronized double diskCleanupKeysPercentage() {
         return ((double) keysToClean.size() / totalKeysInDiskCache) * 100;
     }
 
-    synchronized void cleanUpKeys(Iterator<Key> iterator, Set<CleanupKey> currentKeysToClean, Set<Object> currentFullClean) {
+    synchronized void cleanUpKeys(
+        CachingTier<Key, BytesReference> cachingTier,
+        Set<CleanupKey> currentKeysToClean,
+        Set<Object> currentFullClean
+    ) {
+        Iterator<Key> iterator = cachingTier.keys().iterator();
         while (iterator.hasNext()) {
             Key key = iterator.next();
             CleanupKey cleanupKey = new CleanupKey(key.entity, key.readerCacheKeyId);
             if (currentFullClean.contains(key.entity.getCacheIdentity())) {
-                iterator.remove();
+                cachingTier.invalidate(key);
                 currentFullClean.remove(key.entity.getCacheIdentity());
                 keysToClean.remove(cleanupKey); // since a key could be either in onHeap or disk cache.
             } else if (currentKeysToClean.contains(cleanupKey)) {
-                iterator.remove();
+                cachingTier.invalidate(key);
                 currentKeysToClean.remove(cleanupKey);
                 keysToClean.remove(cleanupKey);
             }

From aa3c39c2bac0d260070d05fdf74ffaed26539f3b Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 28 Nov 2023 22:26:21 -0800
Subject: [PATCH 077/120] change the way we calculate disk key staleness

---
 .../indices/IndicesRequestCache.java          | 89 ++++++++++++++++++-
 1 file changed, 85 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 9fb015e5ca695..2f8d4fe867a59 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -65,6 +65,8 @@
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.index.shard.IndexShard;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -75,6 +77,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 
 /**
@@ -119,6 +122,9 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
 
     private final ConcurrentMap<CleanupKey, Boolean> registeredClosedListeners = ConcurrentCollections.newConcurrentMap();
     private final Set<CleanupKey> keysToClean = ConcurrentCollections.newConcurrentSet();
+    // A map to keep track of the number of keys to be cleaned for a given ShardId and readerCacheKeyId
+    private final ConcurrentMap<ShardId, ConcurrentMap<String , Integer>> diskCleanupKeyToCountMap = ConcurrentCollections.newConcurrentMap();
+    private final AtomicInteger staleKeysInDiskCount = new AtomicInteger(0);
     private final ByteSizeValue size;
     private final TimeValue expire;
     private final TieredCacheService<Key, BytesReference> tieredCacheService;
@@ -196,7 +202,9 @@ public void close() {
     }
 
     void clear(CacheEntity entity) {
-        keysToClean.add(new CleanupKey(entity, null));
+        CleanupKey cleanupKey = new CleanupKey(entity, null);
+        keysToClean.add(cleanupKey);
+        updateStaleKeysInDiskCount(cleanupKey);
         cleanCache();
     }
 
@@ -218,8 +226,71 @@ public void onHit(Key key, CacheValue<BytesReference> cacheValue) {
     @Override
     public void onCached(Key key, BytesReference value, TierType tierType) {
         key.entity.onCached(key, value, tierType);
+        updateDiskCleanupKeyToCountMap(new CleanupKey(key.entity, key.readerCacheKeyId), tierType);
     }
 
+    /**
+     * Updates the diskCleanupKeyToCountMap with the given CleanupKey and TierType.
+     * If the TierType is not DISK, the method returns without making any changes.
+     * If the ShardId associated with the CleanupKey does not exist in the map, a new entry is created.
+     * The method increments the count of the CleanupKey in the map.
+     *
+     * Why use ShardID as the key ?
+     * CacheEntity mainly contains IndexShard, both of these classes do not override equals() and hashCode() methods.
+     * ShardID class properly overrides equals() and hashCode() methods.
+     * Therefore, to avoid modifying CacheEntity and IndexShard classes to override these methods, we use ShardID as the key.
+     *
+     * @param cleanupKey the CleanupKey to be updated in the map
+     * @param tierType the TierType of the CleanupKey
+     */
+    private void updateDiskCleanupKeyToCountMap(CleanupKey cleanupKey, TierType tierType) {
+        if(!tierType.equals(TierType.DISK)) {
+            return;
+        }
+        IndexShard indexShard = (IndexShard)cleanupKey.entity.getCacheIdentity();
+        ShardId shardId = indexShard.shardId();
+
+        diskCleanupKeyToCountMap
+            .computeIfAbsent(shardId, k -> ConcurrentCollections.newConcurrentMap())
+            .merge(cleanupKey.readerCacheKeyId, 1, Integer::sum);
+    }
+
+    /**
+     * Updates the count of stale keys in the disk cache.
+     * This method is called when a CleanupKey is added to the keysToClean set.
+     * It increments the staleKeysInDiskCount by the count of the CleanupKey in the diskCleanupKeyToCountMap.
+     * If the CleanupKey's readerCacheKeyId is null or the CleanupKey's entity is not open, it increments the staleKeysInDiskCount
+     * by the total count of keys associated with the CleanupKey's ShardId in the diskCleanupKeyToCountMap and removes the ShardId from the map.
+     *
+     * @param cleanupKey the CleanupKey that has been marked for cleanup
+     */
+    private void updateStaleKeysInDiskCount(CleanupKey cleanupKey) {
+        IndexShard indexShard = (IndexShard) cleanupKey.entity.getCacheIdentity();
+        ShardId shardId = indexShard.shardId();
+
+        ConcurrentMap<String, Integer> countMap = diskCleanupKeyToCountMap.get(shardId);
+        if (countMap == null) {
+            return;
+        }
+
+        if (cleanupKey.readerCacheKeyId == null) {
+            int totalSum = countMap.values().stream().mapToInt(Integer::intValue).sum();
+            staleKeysInDiskCount.addAndGet(totalSum);
+            diskCleanupKeyToCountMap.remove(shardId);
+            return;
+        }
+        Integer count = countMap.get(cleanupKey.readerCacheKeyId);
+        if (count == null) {
+            return;
+        }
+        staleKeysInDiskCount.addAndGet(count);
+        countMap.remove(cleanupKey.readerCacheKeyId);
+        if (countMap.isEmpty()) {
+            diskCleanupKeyToCountMap.remove(shardId);
+        }
+    }
+
+
     BytesReference getOrCompute(
         CacheEntity cacheEntity,
         CheckedSupplier<BytesReference, IOException> loader,
@@ -414,6 +485,7 @@ public void onClose(IndexReader.CacheKey cacheKey) {
             Boolean remove = registeredClosedListeners.remove(this);
             if (remove != null) {
                 keysToClean.add(this);
+                updateStaleKeysInDiskCount(new CleanupKey(this.entity, this.readerCacheKeyId));
             }
         }
 
@@ -482,7 +554,7 @@ synchronized void cleanDiskCache(double diskCachesCleanThresholdPercent) {
                 return;
             }
             cleanUpKeys(
-                tieredCacheService.getOnHeapCachingTier(),
+                tieredCacheService.getDiskCachingTier().get(),
                 currentKeysToClean,
                 currentFullClean);
         });
@@ -492,10 +564,10 @@ synchronized double diskCleanupKeysPercentage() {
         int totalKeysInDiskCache = tieredCacheService.getDiskCachingTier()
             .map(CachingTier::count)
             .orElse(0);
-        if (totalKeysInDiskCache == 0 || keysToClean.isEmpty()) {
+        if (totalKeysInDiskCache == 0 || staleKeysInDiskCount.get() == 0) {
             return 0;
         }
-        return ((double) keysToClean.size() / totalKeysInDiskCache) * 100;
+        return ((double) staleKeysInDiskCount.get() / totalKeysInDiskCache) * 100;
     }
 
     synchronized void cleanUpKeys(
@@ -543,19 +615,28 @@ long count() {
         return tieredCacheService.count();
     }
 
+    // to be used for testing only
     int numRegisteredCloseListeners() { // for testing
         return registeredClosedListeners.size();
     }
 
+    // to be used for testing only
     void addCleanupKeyForTesting(CacheEntity entity, String readerCacheKeyId) { // for testing
         keysToClean.add(new CleanupKey(entity, readerCacheKeyId));
     }
 
+    // to be used for testing only
     int getKeysToCleanSizeForTesting() { // for testing
         return keysToClean.size();
     }
 
+    // to be used for testing only
     Key createKeyForTesting(CacheEntity entity, String readerCacheKeyId) { // for testing
         return new Key(entity, null, readerCacheKeyId);
     }
+
+    // to be used for testing only
+    void setStaleKeysInDiskCountForTesting(int count) {
+        staleKeysInDiskCount.set(count);
+    }
 }

From 83d70f74ceed288bd66f855c60712511bf5f454a Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 28 Nov 2023 22:27:57 -0800
Subject: [PATCH 078/120] fix breaking tests

---
 .../indices/IndicesRequestCacheTests.java     | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index b323cccc16290..367b0d082f3d5 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -47,15 +47,14 @@
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
 import org.opensearch.action.OriginalIndices;
 import org.opensearch.action.OriginalIndicesTests;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.UUIDs;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.cache.tier.DiskCachingTier;
 import org.opensearch.common.cache.tier.TieredCacheService;
 import org.opensearch.common.cache.tier.TieredCacheSpilloverStrategyService;
@@ -67,7 +66,6 @@
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.AbstractBytesReference;
-import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -79,14 +77,14 @@
 import org.opensearch.index.cache.request.ShardRequestCache;
 import org.opensearch.index.query.TermQueryBuilder;
 import org.opensearch.index.shard.IndexShard;
+import org.opensearch.indices.IndicesRequestCache.CacheEntity;
+import org.opensearch.indices.IndicesRequestCache.Key;
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.SearchShardTarget;
 import org.opensearch.search.internal.AliasFilter;
 import org.opensearch.search.internal.ShardSearchContextId;
 import org.opensearch.search.internal.ShardSearchRequest;
 import org.opensearch.search.query.QuerySearchResult;
-import org.opensearch.indices.IndicesRequestCache.CacheEntity;
-import org.opensearch.indices.IndicesRequestCache.Key;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 
 import java.io.IOException;
@@ -100,7 +98,7 @@
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
-import static org.mockito.internal.verification.VerificationModeFactory.atLeastOnce;
+import static org.mockito.internal.verification.VerificationModeFactory.times;
 
 public class IndicesRequestCacheTests extends OpenSearchSingleNodeTestCase {
 
@@ -700,6 +698,7 @@ public void shouldNotCleanDiskCacheWhenCleanupKeysPercentageIsBelowThreshold() {
         public void cleanDiskCacheWhenCleanupKeysPercentageIsGreaterThanOrEqualToThreshold() {
             final int DISK_CACHE_COUNT = 100;
             final double CLEANUP_THRESHOLD = 50.0;
+            final int STALE_KEYS_IN_DISK_COUNT = 51;
 
             // Mock dependencies
             IndicesService mockIndicesService = mock(IndicesService.class);
@@ -728,11 +727,12 @@ public void cleanDiskCacheWhenCleanupKeysPercentageIsGreaterThanOrEqualToThresho
             when(mockIterator.next()).thenReturn(firstMockKey, secondMockKey);
 
             cache.addCleanupKeyForTesting(mockEntity, "readerCacheKeyId");
+            cache.setStaleKeysInDiskCountForTesting(STALE_KEYS_IN_DISK_COUNT);
             cache.cleanDiskCache(CLEANUP_THRESHOLD);
 
             // Verify interactions
             verify(mockDiskCachingTier).keys();
-            verify(mockIterator, atLeastOnce()).remove();
+            verify(mockIterator, times(2)).next();
         }
 
         @Test
@@ -780,6 +780,7 @@ public void diskCleanupKeysPercentageWhenDiskCacheAndKeysToCleanAreNotEmpty() {
             IndicesRequestCache.CacheEntity mockEntity = Mockito.mock(IndicesRequestCache.CacheEntity.class);
             when(mockEntity.getCacheIdentity()).thenReturn(new Object());
             cache.addCleanupKeyForTesting(mockEntity, "readerCacheKeyId");
+            cache.setStaleKeysInDiskCountForTesting(1);
 
             double result = cache.diskCleanupKeysPercentage();
             assertEquals(1.0, result, 0);

From e41321d4cf156d3ece1007c71e24e3ed7ed9bed8 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 30 Nov 2023 12:50:18 -0800
Subject: [PATCH 079/120] dummy commit

---
 .idea/runConfigurations/Debug_OpenSearch.xml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml
index 0d8bf59823acf..c18046f873477 100644
--- a/.idea/runConfigurations/Debug_OpenSearch.xml
+++ b/.idea/runConfigurations/Debug_OpenSearch.xml
@@ -6,6 +6,10 @@
     <option name="HOST" value="localhost" />
     <option name="PORT" value="5005" />
     <option name="AUTO_RESTART" value="true" />
+    <RunnerSettings RunnerId="Debug">
+      <option name="DEBUG_PORT" value="5005" />
+      <option name="LOCAL" value="false" />
+    </RunnerSettings>
     <method v="2" />
   </configuration>
-</component>
+</component>
\ No newline at end of file

From a51da3890e9cfbada7507b9815a94509fea7b3a9 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 30 Nov 2023 13:12:51 -0800
Subject: [PATCH 080/120] Changed github action to run on my fork. REVERT
 THISgit add .

---
 .github/workflows/assemble.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
index 6a66ac5fb5609..62858a32ab7d4 100644
--- a/.github/workflows/assemble.yml
+++ b/.github/workflows/assemble.yml
@@ -3,7 +3,7 @@ on: [pull_request]
 
 jobs:
   assemble:
-    if: github.repository == 'opensearch-project/OpenSearch'
+    if: github.repository == 'peteralfonsi/OpenSearch'
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:

From f0a26b9aa8991b705d54b7751522218bd88b27b4 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 30 Nov 2023 13:24:22 -0800
Subject: [PATCH 081/120] added broken javadoc

---
 .../org/opensearch/index/cache/request/ShardRequestCache.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index 92c6846b7bbce..9c3570605e93d 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -119,7 +119,7 @@ public void onRemoval(Accountable key, BytesReference value, boolean evicted, Ti
     static abstract class TierStatsAccumulator<S extends TierRequestStats> implements Writeable, ToXContentFragment {
         /**
          * Add new stats from a single request to this holder.
-         * @param stats
+         * @param stats The stats from a single request to add
          */
         public abstract void addRequestStats(S stats);
 

From caae29191a14d97d5d83f4cee041fff8788fd172 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Thu, 30 Nov 2023 13:24:49 -0800
Subject: [PATCH 082/120] Fix test to include IndexShard

---
 .../indices/IndicesRequestCacheTests.java     | 44 ++++++++++++-------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 367b0d082f3d5..d5987956b73b1 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -92,7 +92,6 @@
 import java.util.Iterator;
 import java.util.Optional;
 import java.util.UUID;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
@@ -101,6 +100,12 @@
 import static org.mockito.internal.verification.VerificationModeFactory.times;
 
 public class IndicesRequestCacheTests extends OpenSearchSingleNodeTestCase {
+    public IndexShard getIndexShardCache() {
+        IndexShard indexShard = mock(IndexShard.class);
+        ShardId shardId = mock(ShardId.class);
+        when(indexShard.shardId()).thenReturn(shardId);
+        return indexShard;
+    }
 
     public void testBasicOperationsCache() throws Exception {
         ShardRequestCache requestCacheStats = new ShardRequestCache();
@@ -113,7 +118,7 @@ public void testBasicOperationsCache() throws Exception {
         DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
         TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
         BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        AtomicBoolean indexShard = new AtomicBoolean(true);
+        IndexShard indexShard = getIndexShardCache();
 
         // initial cache
         TestEntity entity = new TestEntity(requestCacheStats, indexShard);
@@ -143,7 +148,7 @@ public void testBasicOperationsCache() throws Exception {
         if (randomBoolean()) {
             reader.close();
         } else {
-            indexShard.set(false); // closed shard but reader is still open
+            entity.setIsOpen(false);
             cache.clear(entity);
         }
         cache.cleanCache();
@@ -161,7 +166,7 @@ public void testBasicOperationsCache() throws Exception {
     public void testCacheDifferentReaders() throws Exception {
         ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
-        AtomicBoolean indexShard = new AtomicBoolean(true);
+        IndexShard indexShard = getIndexShardCache();
         ShardRequestCache requestCacheStats = new ShardRequestCache();
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
@@ -239,7 +244,7 @@ public void testCacheDifferentReaders() throws Exception {
         if (randomBoolean()) {
             secondReader.close();
         } else {
-            indexShard.set(false); // closed shard but reader is still open
+            entity.setIsOpen(false);
             cache.clear(secondEntity);
         }
         cache.cleanCache();
@@ -258,7 +263,7 @@ public void testEviction() throws Exception {
         ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         {
             IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
-            AtomicBoolean indexShard = new AtomicBoolean(true);
+            IndexShard indexShard = getIndexShardCache();
             ShardRequestCache requestCacheStats = new ShardRequestCache();
             Directory dir = newDirectory();
             IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
@@ -287,7 +292,7 @@ public void testEviction() throws Exception {
             getInstanceFromNode(IndicesService.class),
             dummyClusterSettings
         );
-        AtomicBoolean indexShard = new AtomicBoolean(true);
+        IndexShard indexShard = getIndexShardCache();
         ShardRequestCache requestCacheStats = new ShardRequestCache();
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
@@ -316,7 +321,7 @@ public void testEviction() throws Exception {
         logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
         BytesReference value3 = cache.getOrCompute(thirdEntity, thirdLoader, thirdReader, termBytes);
         assertEquals("baz", value3.streamInput().readString());
-        assertEquals(2, cache.count());
+        assertEquals(2, requestCacheStats.stats().getEntries());
         assertEquals(1, requestCacheStats.stats().getEvictions());
         IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
     }
@@ -324,7 +329,7 @@ public void testEviction() throws Exception {
     public void testClearAllEntityIdentity() throws Exception {
         ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
-        AtomicBoolean indexShard = new AtomicBoolean(true);
+        IndexShard indexShard = getIndexShardCache();
 
         ShardRequestCache requestCacheStats = new ShardRequestCache();
         Directory dir = newDirectory();
@@ -344,7 +349,7 @@ public void testClearAllEntityIdentity() throws Exception {
 
         writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
         DirectoryReader thirdReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        AtomicBoolean differentIdentity = new AtomicBoolean(true);
+        IndexShard differentIdentity = getIndexShardCache();
         TestEntity thirdEntity = new TestEntity(requestCacheStats, differentIdentity);
         Loader thirdLoader = new Loader(thirdReader, 0);
 
@@ -417,7 +422,7 @@ public void testInvalidate() throws Exception {
         DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
         TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
         BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        AtomicBoolean indexShard = new AtomicBoolean(true);
+        IndexShard indexShard = getIndexShardCache();
 
         // initial cache
         TestEntity entity = new TestEntity(requestCacheStats, indexShard);
@@ -461,7 +466,7 @@ public void testInvalidate() throws Exception {
         if (randomBoolean()) {
             reader.close();
         } else {
-            indexShard.set(false); // closed shard but reader is still open
+            entity.setIsOpen(false); // closed shard but reader is still open
             cache.clear(entity);
         }
         cache.cleanCache();
@@ -476,8 +481,8 @@ public void testInvalidate() throws Exception {
     }
 
     public void testEqualsKey() throws IOException {
-        AtomicBoolean trueBoolean = new AtomicBoolean(true);
-        AtomicBoolean falseBoolean = new AtomicBoolean(false);
+        IndexShard trueBoolean = getIndexShardCache();
+        IndexShard falseBoolean = getIndexShardCache();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndicesRequestCache indicesRequestCache = indicesService.indicesRequestCache;
         Directory dir = newDirectory();
@@ -617,10 +622,11 @@ public boolean isFragment() {
     }
 
     private class TestEntity extends AbstractIndexShardCacheEntity {
-        private final AtomicBoolean standInForIndexShard;
+        private final IndexShard standInForIndexShard;
         private final ShardRequestCache shardRequestCache;
+        private boolean isOpen = true;
 
-        private TestEntity(ShardRequestCache shardRequestCache, AtomicBoolean standInForIndexShard) {
+        private TestEntity(ShardRequestCache shardRequestCache, IndexShard standInForIndexShard) {
             this.standInForIndexShard = standInForIndexShard;
             this.shardRequestCache = shardRequestCache;
         }
@@ -632,7 +638,11 @@ protected ShardRequestCache stats() {
 
         @Override
         public boolean isOpen() {
-            return standInForIndexShard.get();
+            return this.isOpen;
+        }
+
+        public void setIsOpen(boolean isOpen) {
+            this.isOpen = isOpen;
         }
 
         @Override

From 9d0eab145f3354016d9541a3898d45ad6a19cbc1 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Thu, 30 Nov 2023 13:25:14 -0800
Subject: [PATCH 083/120] UT for testing invalidate of DiskTier is called

---
 .../indices/IndicesRequestCacheTests.java     | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index d5987956b73b1..780a49a13fb34 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -56,6 +56,7 @@
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.UUIDs;
 import org.opensearch.common.cache.tier.DiskCachingTier;
+import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
 import org.opensearch.common.cache.tier.TieredCacheService;
 import org.opensearch.common.cache.tier.TieredCacheSpilloverStrategyService;
 import org.opensearch.common.io.stream.BytesStreamOutput;
@@ -93,6 +94,7 @@
 import java.util.Optional;
 import java.util.UUID;
 
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
@@ -745,6 +747,48 @@ public void cleanDiskCacheWhenCleanupKeysPercentageIsGreaterThanOrEqualToThresho
             verify(mockIterator, times(2)).next();
         }
 
+        @Test
+        public void cleanDiskCacheAndCallInvalidateOfDiskTier() {
+            final int DISK_CACHE_COUNT = 100;
+            final double CLEANUP_THRESHOLD = 50.0;
+            final int STALE_KEYS_IN_DISK_COUNT = 51;
+
+            // Mock dependencies
+            IndicesService mockIndicesService = mock(IndicesService.class);
+            TieredCacheService<Key, BytesReference> mockTieredCacheService = mock(TieredCacheService.class);
+            DiskCachingTier<Key, BytesReference> mockDiskCachingTier = mock(EhCacheDiskCachingTier.class);
+            Iterator<Key> mockIterator = mock(Iterator.class);
+            Iterable<Key> mockIterable = () -> mockIterator;
+
+            IndicesRequestCache cache = new IndicesRequestCache(
+                Settings.EMPTY,
+                mockIndicesService,
+                mockTieredCacheService
+            );
+
+            // Set up mocks
+            when(mockTieredCacheService.getDiskCachingTier()).thenReturn(Optional.of(mockDiskCachingTier));
+            when(mockDiskCachingTier.count()).thenReturn(DISK_CACHE_COUNT);
+            when(mockDiskCachingTier.keys()).thenReturn(mockIterable);
+            when(mockIterator.hasNext()).thenReturn(true, true, false);
+
+            // Create mock Keys and return them when next() is called
+            CacheEntity mockEntity = mock(CacheEntity.class);
+            Key firstMockKey = cache.createKeyForTesting(mockEntity, "readerCacheKeyId1");
+            Key secondMockKey = cache.createKeyForTesting(mockEntity, "readerCacheKeyId2");
+            when(mockEntity.getCacheIdentity()).thenReturn(new Object());
+            when(mockIterator.next()).thenReturn(firstMockKey, secondMockKey);
+
+            cache.addCleanupKeyForTesting(mockEntity, "readerCacheKeyId");
+            cache.setStaleKeysInDiskCountForTesting(STALE_KEYS_IN_DISK_COUNT);
+            cache.cleanDiskCache(CLEANUP_THRESHOLD);
+
+            // Verify interactions
+            verify(mockDiskCachingTier).keys();
+            verify(mockDiskCachingTier, times(1)).invalidate(any(IndicesRequestCache.Key.class));
+            verify(mockIterator, times(2)).next();
+        }
+
         @Test
         public void diskCleanupKeysPercentageWhenDiskCacheIsEmpty() {
             when(tieredCacheService.getDiskCachingTier()).thenReturn(Optional.of(diskCachingTier));

From 01bcad77bffeae99f316546a5ee4c89eead2ab36 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 30 Nov 2023 14:15:25 -0800
Subject: [PATCH 084/120] added upload artifact section to workflow,
 spotlessApply

---
 .github/workflows/assemble.yml                |  5 +++
 .../core/common/bytes/BytesReference.java     |  1 -
 .../IndicesRequestCacheDiskTierIT.java        | 15 ++++++---
 .../indices/IndicesRequestCacheIT.java        |  3 +-
 .../cache/tier/BytesReferenceSerializer.java  |  2 +-
 .../cache/tier/CachePolicyInfoWrapper.java    |  1 +
 .../common/cache/tier/CacheTierPolicy.java    |  4 ---
 .../common/cache/tier/CacheValue.java         |  4 ---
 .../cache/tier/DiskTierRequestStats.java      |  1 +
 .../cache/tier/DiskTierTookTimePolicy.java    |  8 +++--
 .../cache/tier/EhCacheDiskCachingTier.java    | 24 ++++++--------
 .../common/cache/tier/Serializer.java         |  2 --
 .../common/cache/tier/TierType.java           |  1 -
 .../TieredCacheSpilloverStrategyService.java  |  8 +----
 .../tier/keystore/RBMIntKeyLookupStore.java   |  5 +--
 .../cache/request/RequestCacheStats.java      | 24 +++++++-------
 .../cache/request/ShardRequestCache.java      |  8 ++---
 .../index/cache/request/StatsHolder.java      |  4 ---
 .../indices/IRCKeyWriteableSerializer.java    |  5 +--
 .../indices/IndicesRequestCache.java          | 15 ++++-----
 .../opensearch/indices/IndicesService.java    |  1 -
 .../tier/BytesReferenceSerializerTests.java   |  2 +-
 .../tier/DiskTierTookTimePolicyTests.java     |  5 +--
 .../tier/EhCacheDiskCachingTierTests.java     |  7 +++--
 ...redCacheSpilloverStrategyServiceTests.java | 10 ++----
 .../keystore/RBMIntKeyLookupStoreTests.java   | 10 +++---
 .../cache/request/RequestCacheStatsTests.java |  5 ++-
 .../cache/request/ShardRequestCacheTests.java |  2 +-
 .../IRCKeyWriteableSerializerTests.java       | 15 +++++----
 .../indices/IndicesRequestCacheTests.java     | 31 +++++++++++++++----
 30 files changed, 123 insertions(+), 105 deletions(-)

diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
index 62858a32ab7d4..a46ee82acdcdb 100644
--- a/.github/workflows/assemble.yml
+++ b/.github/workflows/assemble.yml
@@ -24,3 +24,8 @@ jobs:
       - name: Run Gradle (assemble)
         run: |
           ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: tiered-caching-artifact
+          path: distribution/archives/linux-tar/build/install/opensearch-3.0.0-SNAPSHOT/lib/opensearch-3.0.0-SNAPSHOT.jar
diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
index e07aca89339b9..9d24d3653397b 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
@@ -45,7 +45,6 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index 4cb6eb1e3c09f..18a37d4d43b42 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -58,7 +58,8 @@ public void testDiskTierStats() throws Exception {
         String node = internalCluster().startNode(
             Settings.builder()
                 .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
-                .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+                .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of
+                                                                                                      // took time
         );
         Client client = client(node);
 
@@ -101,13 +102,13 @@ public void testDiskTierStats() throws Exception {
 
         // We make another actual request that should be in the disk tier. Disk specific stats should keep incrementing
         resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello1")).get();
-        IndicesRequestCacheIT.assertCacheState(client, "index", 0,  numRequests + 2, TierType.ON_HEAP, false);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 2, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 2, numRequests, TierType.DISK, false);
         tookTimeSoFar = assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, -1);
 
         // A final request for something in neither tier shouldn't increment disk specific stats
         resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + numRequests)).get();
-        IndicesRequestCacheIT.assertCacheState(client, "index", 0,  numRequests + 3, TierType.ON_HEAP, false);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 3, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 2, numRequests + 1, TierType.DISK, false);
         assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, tookTimeSoFar);
 
@@ -124,7 +125,13 @@ private long getCacheSizeBytes(Client client, String index, TierType tierType) {
         return requestCacheStats.getMemorySizeInBytes(tierType);
     }
 
-    private long assertDiskTierSpecificStats(Client client, String index, long totalDiskReaches, long totalGetTimeLowerBound, long totalGetTimeUpperBound) {
+    private long assertDiskTierSpecificStats(
+        Client client,
+        String index,
+        long totalDiskReaches,
+        long totalGetTimeLowerBound,
+        long totalGetTimeUpperBound
+    ) {
         // set bounds to -1 to ignore them
         RequestCacheStats requestCacheStats = client.admin()
             .indices()
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index 965afd51d1356..e85cac7cf157c 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -44,7 +44,6 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.common.util.FeatureFlags;
-import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
@@ -677,7 +676,7 @@ public void testCacheWithInvalidation() throws Exception {
         assertCacheState(client, "index", 1, 2, TierType.ON_HEAP, false);
         assertCacheState(client, "index", 0, 2, TierType.DISK, false);
 
-        //assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP); // Evictions won't be 1 until the cache cleaner runs every minute
+        // assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP); // Evictions won't be 1 until the cache cleaner runs every minute
     }
 
     protected static void assertCacheState(
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java b/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
index 201ce7e9c28a6..3ac30b09bddca 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
@@ -11,7 +11,6 @@
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 
-import java.io.IOException;
 import java.util.Arrays;
 
 /**
@@ -22,6 +21,7 @@ public class BytesReferenceSerializer implements Serializer<BytesReference, byte
     // This class does not get passed to ehcache itself, so it's not required that classes match after deserialization.
 
     public BytesReferenceSerializer() {}
+
     @Override
     public byte[] serialize(BytesReference object) {
         return BytesReference.toBytes(object);
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java b/server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java
index 6a077715e8232..ae7854850fed6 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java
@@ -26,6 +26,7 @@
  */
 public class CachePolicyInfoWrapper implements Writeable {
     private final Long tookTimeNanos;
+
     public CachePolicyInfoWrapper(Long tookTimeNanos) {
         this.tookTimeNanos = tookTimeNanos;
         // Add more values here as they are needed for future cache tier policies
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
index dc884721d8e14..ac5fbb230d173 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
@@ -32,10 +32,6 @@
 
 package org.opensearch.common.cache.tier;
 
-import org.opensearch.core.common.bytes.BytesReference;
-
-import java.io.IOException;
-
 /**
  * An interface for policies that inspect data of type T to decide whether they are admitted into a cache tier.
  */
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java b/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
index baeac79c172d9..bb3735e48b181 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
@@ -8,10 +8,6 @@
 
 package org.opensearch.common.cache.tier;
 
-import org.opensearch.common.cache.tier.TierType;
-
-import java.util.Map;
-
 /**
  * Represents a cache value, with its associated tier type where it is stored,
  * and tier-specific stats for an individual request stored in a map.
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java
index 33f36202c3f84..9c2d1c5d1a706 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java
@@ -20,6 +20,7 @@ public DiskTierRequestStats(long requestGetTimeNanos, boolean requestReachedDisk
         this.requestReachedDisk = requestReachedDisk;
         this.requestGetTimeNanos = requestGetTimeNanos;
     }
+
     @Override
     public TierType getTierType() {
         return TierType.DISK;
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
index 7de6583c485ed..d6c3b894a974d 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
@@ -18,9 +18,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.search.query.QuerySearchResult;
 
-import java.io.IOException;
 import java.util.function.Function;
 
 /**
@@ -40,7 +38,11 @@ public class DiskTierTookTimePolicy implements CacheTierPolicy<BytesReference> {
     private TimeValue threshold;
     private final Function<BytesReference, CachePolicyInfoWrapper> getPolicyInfoFn;
 
-    public DiskTierTookTimePolicy(Settings settings, ClusterSettings clusterSettings, Function<BytesReference, CachePolicyInfoWrapper> getPolicyInfoFn) {
+    public DiskTierTookTimePolicy(
+        Settings settings,
+        ClusterSettings clusterSettings,
+        Function<BytesReference, CachePolicyInfoWrapper> getPolicyInfoFn
+    ) {
         this.threshold = DISK_TOOKTIME_THRESHOLD_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(DISK_TOOKTIME_THRESHOLD_SETTING, this::setThreshold);
         this.getPolicyInfoFn = getPolicyInfoFn;
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index f8aedbe5314de..aa136b02cbfb7 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -8,8 +8,6 @@
 
 package org.opensearch.common.cache.tier;
 
-import org.ehcache.core.spi.service.FileBasedPersistenceContext;
-import org.ehcache.spi.serialization.SerializerException;
 import org.opensearch.OpenSearchException;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
@@ -21,12 +19,9 @@
 import org.opensearch.common.unit.TimeValue;
 
 import java.io.File;
-import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.time.Duration;
-import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Objects;
 import java.util.Optional;
@@ -41,14 +36,13 @@
 import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
 import org.ehcache.config.builders.ResourcePoolsBuilder;
 import org.ehcache.config.units.MemoryUnit;
+import org.ehcache.core.spi.service.FileBasedPersistenceContext;
 import org.ehcache.event.CacheEvent;
 import org.ehcache.event.CacheEventListener;
 import org.ehcache.event.EventType;
 import org.ehcache.expiry.ExpiryPolicy;
 import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
-import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.unit.ByteSizeValue;
-import org.opensearch.search.query.QuerySearchResult;
+import org.ehcache.spi.serialization.SerializerException;
 
 /**
  * An ehcache-based disk tier implementation.
@@ -133,7 +127,7 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         close();
         cacheManager = buildCacheManager();
         this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
-        
+
         long keystoreMaxWeight = builder.keystoreMaxWeightInBytes;
         this.keystore = new RBMIntKeyLookupStore(keystoreMaxWeight);
     }
@@ -177,11 +171,11 @@ public Duration getExpiryForUpdate(K key, Supplier<? extends byte[]> oldValue, b
             })
                 .withService(getListenerConfiguration(builder))
                 .withService(
-                new OffHeapDiskStoreConfiguration(
-                    this.threadPoolAlias,
-                    DISK_WRITE_CONCURRENCY.get(settings),
-                    DISK_SEGMENTS.get(settings)
-                )
+                    new OffHeapDiskStoreConfiguration(
+                        this.threadPoolAlias,
+                        DISK_WRITE_CONCURRENCY.get(settings),
+                        DISK_SEGMENTS.get(settings)
+                    )
                 )
                 .withKeySerializer(new KeySerializerWrapper<K>(keySerializer))
         );
@@ -383,6 +377,7 @@ public K next() {
      */
     private class KeySerializerWrapper<K> implements org.ehcache.spi.serialization.Serializer<K> {
         public Serializer<K, byte[]> serializer;
+
         public KeySerializerWrapper(Serializer<K, byte[]> serializer) {
             this.serializer = serializer;
         }
@@ -440,6 +435,7 @@ public static class Builder<K, V> {
         private Serializer<K, byte[]> keySerializer;
         private Serializer<V, byte[]> valueSerializer;
         private long keystoreMaxWeightInBytes = 0;
+
         public Builder() {}
 
         public EhCacheDiskCachingTier.Builder<K, V> setMaximumWeightInBytes(long sizeInBytes) {
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/Serializer.java b/server/src/main/java/org/opensearch/common/cache/tier/Serializer.java
index 74c256d188682..04413fce4b8b0 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/Serializer.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/Serializer.java
@@ -8,8 +8,6 @@
 
 package org.opensearch.common.cache.tier;
 
-import java.io.IOException;
-
 /**
  * An interface for serializers, to be used in disk caching tier and elsewhere.
  * T is the class of the original object, and U is the serialized class.
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TierType.java b/server/src/main/java/org/opensearch/common/cache/tier/TierType.java
index e3c18f6a1b5a3..3ef4338030899 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TierType.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TierType.java
@@ -27,4 +27,3 @@ public String getStringValue() {
         return this.stringValue;
     }
 }
-
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 78e4cc5e11f48..a0a24ef28764b 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -8,13 +8,9 @@
 
 package org.opensearch.common.cache.tier;
 
-import org.opensearch.common.cache.Cache;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
-import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.indices.IndicesRequestCache;
-import org.opensearch.search.query.QuerySearchResult;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -187,7 +183,7 @@ private Function<K, CacheValue<V>> getValueFromTierCache(boolean trackStats) {
                     if (trackStats) {
                         tieredCacheEventListener.onHit(key, cacheValue);
                     }
-                    return cacheValue; //new CacheValue<>(value, cachingTier.getTierType());
+                    return cacheValue; // new CacheValue<>(value, cachingTier.getTierType());
                 }
                 if (trackStats) {
                     tieredCacheEventListener.onMiss(key, cacheValue);
@@ -197,8 +193,6 @@ private Function<K, CacheValue<V>> getValueFromTierCache(boolean trackStats) {
         };
     }
 
-
-
     /**
      * Builder object
      * @param <K> Type of key
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index 405cbd435a52d..bb258439de83e 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -218,7 +218,8 @@ public boolean remove(Integer value) {
                 if (removalSet == null) {
                     // First time a removal has been attempted for this transformed value
                     HashSet<Integer> newRemovalSet = new HashSet<>();
-                    newRemovalSet.add(value); // Add the key value, not the transformed value, to the list of attempted removals for this transformedValue
+                    newRemovalSet.add(value); // Add the key value, not the transformed value, to the list of attempted removals for this
+                                              // transformedValue
                     removalSets.put(transformedValue, newRemovalSet);
                     numCollisions.dec();
                 } else {
@@ -280,7 +281,7 @@ public boolean isCollision(Integer value1, Integer value2) {
         return transform(value1) == transform(value2);
     }
 
-     static double getRBMSizeMultiplier(int numEntries, int modulo) {
+    static double getRBMSizeMultiplier(int numEntries, int modulo) {
         double effectiveModulo = (double) modulo / 2;
         /* This model was created when we used % operator to calculate modulo. This has range (-modulo, modulo).
         Now we have optimized to use a bitmask, which has range [0, modulo). So the number of possible values stored
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index 479c84d1b5254..4836479133b67 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -34,7 +34,6 @@
 
 import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
-import org.opensearch.common.cache.tier.TierRequestStats;
 import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,7 +43,6 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -55,17 +53,21 @@
  */
 public class RequestCacheStats implements Writeable, ToXContentFragment {
 
-    private Map<String, StatsHolder> defaultStatsMap = new HashMap<>(){{
-        for (TierType tierType : TierType.values()) {
-            put(tierType.getStringValue(), new StatsHolder());
-            // Every possible tier type must have counters, even if they are disabled. Then the counters report 0
-        }}
+    private Map<String, StatsHolder> defaultStatsMap = new HashMap<>() {
+        {
+            for (TierType tierType : TierType.values()) {
+                put(tierType.getStringValue(), new StatsHolder());
+                // Every possible tier type must have counters, even if they are disabled. Then the counters report 0
+            }
+        }
     };
 
-     private Map<String, ShardRequestCache.TierStatsAccumulator> tierSpecificStatsMap = new HashMap<>(){{
-        put(TierType.ON_HEAP.getStringValue(), new ShardRequestCache.OnHeapStatsAccumulator());
-        put(TierType.DISK.getStringValue(), new ShardRequestCache.DiskStatsAccumulator());
-     }};
+    private Map<String, ShardRequestCache.TierStatsAccumulator> tierSpecificStatsMap = new HashMap<>() {
+        {
+            put(TierType.ON_HEAP.getStringValue(), new ShardRequestCache.OnHeapStatsAccumulator());
+            put(TierType.DISK.getStringValue(), new ShardRequestCache.DiskStatsAccumulator());
+        }
+    };
 
     public RequestCacheStats() {}
 
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index 9c3570605e93d..67f920850e949 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -137,7 +137,9 @@ static abstract class TierStatsAccumulator<S extends TierRequestStats> implement
      */
     public static class OnHeapStatsAccumulator extends TierStatsAccumulator<OnHeapTierRequestStats> {
         OnHeapStatsAccumulator() {}
+
         OnHeapStatsAccumulator(StreamInput in) {}
+
         @Override
         public void addRequestStats(OnHeapTierRequestStats stats) {}
 
@@ -159,6 +161,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     public static class DiskStatsAccumulator extends TierStatsAccumulator<DiskTierRequestStats> {
         final CounterMetric totalGetTime;
         final CounterMetric totalDiskReaches; // Number of times a get() has actually reached the disk
+
         public DiskStatsAccumulator() {
             this.totalGetTime = new CounterMetric();
             this.totalDiskReaches = new CounterMetric();
@@ -172,10 +175,7 @@ public DiskStatsAccumulator(long totalGetTime, long totalDiskReaches) {
         }
 
         public DiskStatsAccumulator(StreamInput in) throws IOException {
-            this(
-                in.readVLong(),
-                in.readVLong()
-            );
+            this(in.readVLong(), in.readVLong());
         }
 
         public long getTotalGetTime() {
diff --git a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
index 300689cf09dff..74e610cf6d9cc 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
@@ -8,8 +8,6 @@
 
 package org.opensearch.index.cache.request;
 
-import org.opensearch.common.cache.tier.TierRequestStats;
-import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -20,8 +18,6 @@
 
 import java.io.IOException;
 import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
 
 /**
  * The basic StatsHolder class, which accumulates shard-level stats that are common across all tiers.
diff --git a/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java b/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
index 63a3447d481c0..92e874518df54 100644
--- a/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
+++ b/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
@@ -9,13 +9,12 @@
 package org.opensearch.indices;
 
 import org.opensearch.OpenSearchException;
+import org.opensearch.common.cache.tier.Serializer;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.BytesStreamInput;
-import org.opensearch.common.cache.tier.Serializer;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.Arrays;
 
 /**
@@ -24,9 +23,11 @@
 public class IRCKeyWriteableSerializer implements Serializer<IndicesRequestCache.Key, byte[]> {
 
     IndicesRequestCache irc;
+
     public IRCKeyWriteableSerializer(IndicesRequestCache irc) {
         this.irc = irc;
     }
+
     @Override
     public byte[] serialize(IndicesRequestCache.Key object) {
         try {
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 2fbdd43abb23d..cb253ca6a1ed6 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -42,8 +42,8 @@
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.tier.BytesReferenceSerializer;
 import org.opensearch.common.cache.tier.CachePolicyInfoWrapper;
-import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
 import org.opensearch.common.cache.tier.CacheValue;
+import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
 import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
 import org.opensearch.common.cache.tier.OnHeapCachingTier;
 import org.opensearch.common.cache.tier.OpenSearchOnHeapCache;
@@ -64,7 +64,6 @@
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.common.unit.ByteSizeValue;
-import org.opensearch.search.query.QuerySearchResult;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -149,8 +148,9 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         double diskTierKeystoreWeightFraction = 0.05; // Allocate 5% of the on-heap weight to the disk tier's keystore
         long keystoreMaxWeight = (long) (diskTierKeystoreWeightFraction * INDICES_CACHE_QUERY_SIZE.get(settings).getBytes());
 
-        EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
-            .setKeyType(Key.class)
+        EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>().setKeyType(
+            Key.class
+        )
             .setValueType(BytesReference.class)
             .setExpireAfterAccess(TimeValue.MAX_VALUE)
             .setSettings(settings)
@@ -164,8 +164,9 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             .build();
 
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
-        tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>()
-            .setOnHeapCachingTier(openSearchOnHeapCache)
+        tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>().setOnHeapCachingTier(
+            openSearchOnHeapCache
+        )
             .setOnDiskCachingTier(ehcacheDiskTier)
             .setTieredCacheEventListener(this)
             .withPolicy(new DiskTierTookTimePolicy(settings, clusterSettings, transformationFunction))
@@ -328,7 +329,7 @@ interface CacheEntity extends Accountable, Writeable {
      *
      * @opensearch.internal
      */
-     class Key implements Accountable, Writeable {
+    class Key implements Accountable, Writeable {
         private final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Key.class);
 
         public final CacheEntity entity; // use as identity equality
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index f513f067bd9be..c8150103b29b5 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -61,7 +61,6 @@
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.Nullable;
-import org.opensearch.common.cache.Cache;
 import org.opensearch.common.cache.tier.CachePolicyInfoWrapper;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lease.Releasable;
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/BytesReferenceSerializerTests.java b/server/src/test/java/org/opensearch/common/cache/tier/BytesReferenceSerializerTests.java
index 9b9927978070a..af81f04149ae6 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/BytesReferenceSerializerTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/BytesReferenceSerializerTests.java
@@ -34,7 +34,7 @@ public void testEquality() throws Exception {
         BytesReference deserialized = ser.deserialize(serialized);
         assertEquals(ba, deserialized);
 
-        ba = new BytesArray(new byte[]{});
+        ba = new BytesArray(new byte[] {});
         serialized = ser.serialize(ba);
         assertTrue(ser.equals(ba, serialized));
         deserialized = ser.deserialize(serialized);
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java b/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
index 9705a3fae4e24..f7df259ff44d2 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicyTests.java
@@ -46,7 +46,6 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.indices.IndicesRequestCache;
 import org.opensearch.search.DocValueFormat;
@@ -68,6 +67,7 @@ public class DiskTierTookTimePolicyTests extends OpenSearchTestCase {
             throw new RuntimeException(e);
         }
     };
+
     private DiskTierTookTimePolicy getTookTimePolicy() {
         // dummy settings
         Settings dummySettings = Settings.EMPTY;
@@ -80,6 +80,7 @@ public void testQSRSetupFunction() throws IOException {
         QuerySearchResult qsr = getQSR(ttn);
         assertEquals(ttn, qsr.getTookTimeNanos());
     }
+
     public void testTookTimePolicy() throws Exception {
         DiskTierTookTimePolicy tookTimePolicy = getTookTimePolicy();
 
@@ -104,7 +105,7 @@ public void testTookTimePolicy() throws Exception {
         assertTrue(longResult);
     }
 
-     public static QuerySearchResult getQSR(long tookTimeNanos) {
+    public static QuerySearchResult getQSR(long tookTimeNanos) {
         // package-private, also used by IndicesRequestCacheTests.java
         // setup from QuerySearchResultTests.java
         ShardId shardId = new ShardId("index", "uuid", randomInt());
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
index 45928b2f20610..139e82b274d34 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
@@ -10,7 +10,6 @@
 
 import org.opensearch.common.Randomness;
 import org.opensearch.common.cache.RemovalListener;
-import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.bytes.BytesArray;
@@ -72,8 +71,9 @@ public void testBasicGetAndPut() throws IOException {
     public void testBasicGetAndPutBytesReference() throws Exception {
         Settings settings = Settings.builder().build();
         try (NodeEnvironment env = newNodeEnvironment(settings)) {
-            EhCacheDiskCachingTier<String, BytesReference> ehCacheDiskCachingTier = new EhCacheDiskCachingTier.Builder<String, BytesReference>()
-                .setKeyType(String.class)
+            EhCacheDiskCachingTier<String, BytesReference> ehCacheDiskCachingTier = new EhCacheDiskCachingTier.Builder<
+                String,
+                BytesReference>().setKeyType(String.class)
                 .setValueType(BytesReference.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setSettings(settings)
@@ -317,6 +317,7 @@ private RemovalListener<String, String> removalListener(AtomicInteger counter) {
     private static class StringSerializer implements Serializer<String, byte[]> {
 
         private final Charset charset = StandardCharsets.UTF_8;
+
         @Override
         public byte[] serialize(String object) {
             return object.getBytes(charset);
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java b/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
index f0b286b13b200..855ab1be6b1e5 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyServiceTests.java
@@ -15,12 +15,10 @@
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
@@ -34,7 +32,7 @@ public void testComputeAndAbsentWithoutAnyOnHeapCacheEviction() throws Exception
             onHeapCacheSize,
             randomIntBetween(1, 4),
             eventListener,
-                null
+            null
         );
         int numOfItems1 = randomIntBetween(1, onHeapCacheSize / 2 - 1);
         List<String> keys = new ArrayList<>();
@@ -209,9 +207,7 @@ public void testWithDiskTierNull() throws Exception {
         Function<String, String> identityFunction = (String value) -> { return value; };
         TieredCacheSpilloverStrategyService<String, String> spilloverStrategyService = new TieredCacheSpilloverStrategyService.Builder<
             String,
-            String>().setOnHeapCachingTier(new MockOnHeapCacheTier<>(onHeapCacheSize))
-            .setTieredCacheEventListener(eventListener)
-            .build();
+            String>().setOnHeapCachingTier(new MockOnHeapCacheTier<>(onHeapCacheSize)).setTieredCacheEventListener(eventListener).build();
         int numOfItems = randomIntBetween(onHeapCacheSize + 1, onHeapCacheSize * 3);
         for (int iter = 0; iter < numOfItems; iter++) {
             TieredCacheLoader<String, String> tieredCacheLoader = getTieredCacheLoader();
@@ -307,6 +303,7 @@ public boolean isLoaded() {
     private TieredCacheLoader<String, String> getTieredCacheLoaderWithKeyValueMap(Map<String, String> map) {
         return new TieredCacheLoader<String, String>() {
             boolean isLoaded;
+
             @Override
             public String load(String key) throws Exception {
                 isLoaded = true;
@@ -334,7 +331,6 @@ private TieredCacheSpilloverStrategyService<String, String> intializeTieredCache
             policiesToUse = policies;
         }
 
-
         return new TieredCacheSpilloverStrategyService.Builder<String, String>().setOnHeapCachingTier(openSearchOnHeapCache)
             .setOnDiskCachingTier(diskCache)
             .setTieredCacheEventListener(cacheEventListener)
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index d9b1ece1310ca..b1ec815980c23 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -34,7 +34,6 @@
 import org.opensearch.common.Randomness;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.test.OpenSearchTestCase;
-import org.roaringbitmap.RoaringBitmap;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -43,9 +42,12 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.ThreadPoolExecutor;
 
+import org.roaringbitmap.RoaringBitmap;
+
 public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase {
 
     final int BYTES_IN_MB = 1048576;
+
     public void testInit() {
         long memCap = 100 * BYTES_IN_MB;
         RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(memCap);
@@ -151,10 +153,10 @@ public void testMemoryCapBlocksAdd() throws Exception {
         // as that's what our size estimator is designed for.
         // If we add a run of integers, our size estimator is not valid, especially for small RBMs.
 
-        int[] maxEntriesArr = new int[] { 1342000, 100000, 3000000};
+        int[] maxEntriesArr = new int[] { 1342000, 100000, 3000000 };
         long[] rbmReportedSizes = new long[4];
         Random rand = Randomness.get();
-        for (int j = 0; j < maxEntriesArr.length; j++)  {
+        for (int j = 0; j < maxEntriesArr.length; j++) {
             RoaringBitmap rbm = new RoaringBitmap();
             for (int i = 0; i < maxEntriesArr[j]; i++) {
                 rbm.add(rand.nextInt());
@@ -165,7 +167,7 @@ public void testMemoryCapBlocksAdd() throws Exception {
         for (int i = 0; i < maxEntriesArr.length; i++) {
             double multiplier = RBMIntKeyLookupStore.getRBMSizeMultiplier(maxEntriesArr[i], moduloValue.getValue());
             long memSizeCapInBytes = (long) (rbmReportedSizes[i] * multiplier);
-            //long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue);
+            // long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue);
             RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes);
             for (int j = 0; j < maxEntriesArr[i] + 5000; j++) {
                 kls.add(rand.nextInt());
diff --git a/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java b/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
index b481cf762f9ec..54268d368f2e9 100644
--- a/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
+++ b/server/src/test/java/org/opensearch/index/cache/request/RequestCacheStatsTests.java
@@ -92,6 +92,9 @@ private void assertTierState(
 
     private void assertDiskStatsState(RequestCacheStats stats, long totalGetTime, long totalDiskReaches) {
         assertEquals(totalGetTime, ((ShardRequestCache.DiskStatsAccumulator) stats.getTierSpecificStats(TierType.DISK)).getTotalGetTime());
-        assertEquals(totalDiskReaches, ((ShardRequestCache.DiskStatsAccumulator) stats.getTierSpecificStats(TierType.DISK)).getTotalDiskReaches());
+        assertEquals(
+            totalDiskReaches,
+            ((ShardRequestCache.DiskStatsAccumulator) stats.getTierSpecificStats(TierType.DISK)).getTotalDiskReaches()
+        );
     }
 }
diff --git a/server/src/test/java/org/opensearch/index/cache/request/ShardRequestCacheTests.java b/server/src/test/java/org/opensearch/index/cache/request/ShardRequestCacheTests.java
index 8b97924343152..9d55ff3a5c5f2 100644
--- a/server/src/test/java/org/opensearch/index/cache/request/ShardRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/index/cache/request/ShardRequestCacheTests.java
@@ -8,7 +8,6 @@
 
 package org.opensearch.index.cache.request;
 
-
 import org.opensearch.common.cache.tier.DiskTierRequestStats;
 import org.opensearch.test.OpenSearchTestCase;
 
@@ -20,6 +19,7 @@ public void testInit() throws Exception {
         ShardRequestCache src = new ShardRequestCache();
         RequestCacheStats rcs = src.stats();
     }
+
     public void testDiskStatsAccumulator() throws Exception {
         ShardRequestCache.DiskStatsAccumulator acc = new ShardRequestCache.DiskStatsAccumulator();
         DiskTierRequestStats reachedDiskReqStats = new DiskTierRequestStats(145L, true);
diff --git a/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java b/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
index c18250bb6bec2..723f8927d076d 100644
--- a/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
+++ b/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
@@ -15,9 +15,7 @@
 import org.opensearch.index.IndexService;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
-import org.opensearch.test.OpenSearchTestCase;
 
-import java.nio.ByteBuffer;
 import java.util.Random;
 import java.util.UUID;
 
@@ -33,9 +31,9 @@ public void testSerializer() throws Exception {
         IRCKeyWriteableSerializer ser = new IRCKeyWriteableSerializer(irc);
 
         int NUM_KEYS = 1000;
-        int[] valueLengths = new int[]{ 1000, 6000 }; // test both branches in equals()
+        int[] valueLengths = new int[] { 1000, 6000 }; // test both branches in equals()
         Random rand = Randomness.get();
-        for (int valueLength: valueLengths) {
+        for (int valueLength : valueLengths) {
             for (int i = 0; i < NUM_KEYS; i++) {
                 IndicesRequestCache.Key key = getRandomIRCKey(valueLength, rand, irc, entity);
                 byte[] serialized = ser.serialize(key);
@@ -45,7 +43,13 @@ public void testSerializer() throws Exception {
             }
         }
     }
-    private IndicesRequestCache.Key getRandomIRCKey(int valueLength, Random random, IndicesRequestCache irc, IndicesService.IndexShardCacheEntity entity) {
+
+    private IndicesRequestCache.Key getRandomIRCKey(
+        int valueLength,
+        Random random,
+        IndicesRequestCache irc,
+        IndicesService.IndexShardCacheEntity entity
+    ) {
         byte[] value = new byte[valueLength];
         for (int i = 0; i < valueLength; i++) {
             value[i] = (byte) (random.nextInt(126 - 32) + 32);
@@ -54,4 +58,3 @@ private IndicesRequestCache.Key getRandomIRCKey(int valueLength, Random random,
         return irc.new Key(entity, keyValue, UUID.randomUUID().toString()); // same UUID source as used in real key
     }
 }
-
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index aee54dcf3379c..8c49fc36076e2 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -60,7 +60,6 @@
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.AbstractBytesReference;
-import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -90,7 +89,11 @@ public class IndicesRequestCacheTests extends OpenSearchSingleNodeTestCase {
     public void testBasicOperationsCache() throws Exception {
         ShardRequestCache requestCacheStats = new ShardRequestCache();
         ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
+        IndicesRequestCache cache = new IndicesRequestCache(
+            Settings.EMPTY,
+            getInstanceFromNode(IndicesService.class),
+            dummyClusterSettings
+        );
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
 
@@ -145,7 +148,11 @@ public void testBasicOperationsCache() throws Exception {
 
     public void testCacheDifferentReaders() throws Exception {
         ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
+        IndicesRequestCache cache = new IndicesRequestCache(
+            Settings.EMPTY,
+            getInstanceFromNode(IndicesService.class),
+            dummyClusterSettings
+        );
         AtomicBoolean indexShard = new AtomicBoolean(true);
         ShardRequestCache requestCacheStats = new ShardRequestCache();
         Directory dir = newDirectory();
@@ -242,7 +249,11 @@ public void testEviction() throws Exception {
         final ByteSizeValue size;
         ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         {
-            IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
+            IndicesRequestCache cache = new IndicesRequestCache(
+                Settings.EMPTY,
+                getInstanceFromNode(IndicesService.class),
+                dummyClusterSettings
+            );
             AtomicBoolean indexShard = new AtomicBoolean(true);
             ShardRequestCache requestCacheStats = new ShardRequestCache();
             Directory dir = newDirectory();
@@ -308,7 +319,11 @@ public void testEviction() throws Exception {
 
     public void testClearAllEntityIdentity() throws Exception {
         ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
+        IndicesRequestCache cache = new IndicesRequestCache(
+            Settings.EMPTY,
+            getInstanceFromNode(IndicesService.class),
+            dummyClusterSettings
+        );
         AtomicBoolean indexShard = new AtomicBoolean(true);
 
         ShardRequestCache requestCacheStats = new ShardRequestCache();
@@ -394,7 +409,11 @@ public BytesReference get() {
     public void testInvalidate() throws Exception {
         ShardRequestCache requestCacheStats = new ShardRequestCache();
         ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, getInstanceFromNode(IndicesService.class), dummyClusterSettings);
+        IndicesRequestCache cache = new IndicesRequestCache(
+            Settings.EMPTY,
+            getInstanceFromNode(IndicesService.class),
+            dummyClusterSettings
+        );
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
 

From c6a93a424eb54ac390567951aadfcd4466ea7b5d Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Fri, 1 Dec 2023 09:26:59 -0800
Subject: [PATCH 085/120] Added notification for REMOVED case in ehcache event
 listener

---
 .../common/cache/tier/EhCacheDiskCachingTier.java      | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index aa136b02cbfb7..639936b62fc7b 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -320,6 +320,16 @@ public void onEvent(CacheEvent<? extends K, ? extends byte[]> event) {
                     assert event.getNewValue() == null;
                     break;
                 case REMOVED:
+                    this.removalListener.ifPresent(
+                        listener -> listener.onRemoval(
+                            new RemovalNotification<>(
+                                event.getKey(),
+                                valueSerializer.deserialize(event.getOldValue()),
+                                RemovalReason.INVALIDATED,
+                                TierType.DISK
+                            )
+                        )
+                    );
                     count.dec();
                     assert event.getNewValue() == null;
                     break;

From 93ef1247e7742846ffd5f3f1e6feab14203f320d Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 4 Dec 2023 14:06:56 -0800
Subject: [PATCH 086/120] draft changes for enable disk tier setting

---
 .../cache/tier/DiskTierTookTimePolicy.java    |   2 +-
 .../common/cache/tier/TieredCacheService.java |   2 +
 .../TieredCacheSpilloverStrategyService.java  |  27 ++++-
 .../indices/IndicesRequestCache.java          | 100 ++++++++++++++----
 4 files changed, 107 insertions(+), 24 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
index d6c3b894a974d..d7977d6a1ca30 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
@@ -29,7 +29,7 @@
  */
 public class DiskTierTookTimePolicy implements CacheTierPolicy<BytesReference> {
     public static final Setting<TimeValue> DISK_TOOKTIME_THRESHOLD_SETTING = Setting.positiveTimeSetting(
-        "index.requests.cache.disk.tooktime.threshold",
+        "index.requests.cache.tiered.disk.took_time_threshold",
         new TimeValue(10),
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
index 31d58510206f0..83c76656786bc 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
@@ -31,4 +31,6 @@ public interface TieredCacheService<K, V> {
     OnHeapCachingTier<K, V> getOnHeapCachingTier();
 
     Optional<DiskCachingTier<K, V>> getDiskCachingTier();
+    void addDiskTier(DiskCachingTier<K, V> newTier);
+    void removeDiskTier();
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index a0a24ef28764b..e18c15f7ce063 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -11,6 +11,10 @@
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.indices.IndicesRequestCache;
+import org.opensearch.indices.IndicesService;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -32,13 +36,13 @@ public class TieredCacheSpilloverStrategyService<K, V> implements TieredCacheSer
     /**
      * Optional in case tiered caching is turned off.
      */
-    private final Optional<DiskCachingTier<K, V>> diskCachingTier;
+    private Optional<DiskCachingTier<K, V>> diskCachingTier;
     private final TieredCacheEventListener<K, V> tieredCacheEventListener;
 
     /**
      * Maintains caching tiers in order of get calls.
      */
-    private final List<CachingTier<K, V>> cachingTierList;
+    private List<CachingTier<K, V>> cachingTierList;
     private final List<CacheTierPolicy<V>> policies;
 
     private TieredCacheSpilloverStrategyService(Builder<K, V> builder) {
@@ -193,6 +197,25 @@ private Function<K, CacheValue<V>> getValueFromTierCache(boolean trackStats) {
         };
     }
 
+    /**
+     * Dynamically add a new disk tier.
+     */
+    public void addDiskTier(DiskCachingTier<K, V> newTier) {
+        assert getDiskCachingTier().isEmpty();
+        diskCachingTier = Optional.of(newTier);
+        cachingTierList = List.of(onHeapCachingTier, diskCachingTier.get());
+    }
+
+    /**
+     * Dynamically remove an existing disk tier.
+     */
+    public void removeDiskTier() {
+        assert getDiskCachingTier().isPresent();
+        diskCachingTier.get().close();
+        diskCachingTier = Optional.empty();
+        cachingTierList = List.of(onHeapCachingTier);
+    }
+
     /**
      * Builder object
      * @param <K> Type of key
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index cb253ca6a1ed6..4f91d11832565 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -63,6 +63,7 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 
 import java.io.Closeable;
@@ -71,7 +72,9 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.Function;
@@ -116,12 +119,40 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         Property.NodeScope
     );
 
+    public static final Setting<Boolean> INDICES_CACHE_DISK_TIER_ENABLED = Setting.boolSetting(
+        "index.requests.cache.tiered.disk.enable",
+        false,
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
+    public static final Setting<ByteSizeValue> INDICES_CACHE_DISK_TIER_SIZE = Setting.memorySizeSetting(
+        "index.requests.cache.tiered.disk.size",
+        new ByteSizeValue(1, ByteSizeUnit.GB), // TODO: This is a random value - what should it be??
+        Property.NodeScope
+    );
+
+    public static final Setting<Double> INDICES_CACHE_DISK_STALE_KEY_THRESHOLD = Setting.doubleSetting(
+        "index.requests.cache.tiered.disk.stale_cleanup_threshold",
+        0.5,
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
+    public static final Setting<ByteSizeValue> INDICES_CACHE_KEYSTORE_SIZE = Setting.memorySizeSetting(
+        "index.requests.cache.tiered.disk.keystore.size",
+        "0.05%", // 5% of INDICES_CACHE_QUERY_SIZE
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
     private final ConcurrentMap<CleanupKey, Boolean> registeredClosedListeners = ConcurrentCollections.newConcurrentMap();
     private final Set<CleanupKey> keysToClean = ConcurrentCollections.newConcurrentSet();
     private final ByteSizeValue size;
     private final TimeValue expire;
     private final TieredCacheService<Key, BytesReference> tieredCacheService;
     private final IndicesService indicesService;
+    private final Settings settings;
 
     IndicesRequestCache(Settings settings, IndicesService indicesService, ClusterSettings clusterSettings) {
         this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
@@ -141,27 +172,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             }
         };
         // enabling this for testing purposes. Remove/tweak!!
-        long CACHE_SIZE_IN_BYTES = 1000000L;
-        String SETTING_PREFIX = "indices.request.cache";
-        String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
-
-        double diskTierKeystoreWeightFraction = 0.05; // Allocate 5% of the on-heap weight to the disk tier's keystore
-        long keystoreMaxWeight = (long) (diskTierKeystoreWeightFraction * INDICES_CACHE_QUERY_SIZE.get(settings).getBytes());
-
-        EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>().setKeyType(
-            Key.class
-        )
-            .setValueType(BytesReference.class)
-            .setExpireAfterAccess(TimeValue.MAX_VALUE)
-            .setSettings(settings)
-            .setThreadPoolAlias("ehcacheTest")
-            .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
-            .setStoragePath(STORAGE_PATH)
-            .setSettingPrefix(SETTING_PREFIX)
-            .setKeySerializer(new IRCKeyWriteableSerializer(this))
-            .setValueSerializer(new BytesReferenceSerializer())
-            .setKeyStoreMaxWeightInBytes(keystoreMaxWeight)
-            .build();
+        EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = getDefaultDiskTier(settings);
 
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
         tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>().setOnHeapCachingTier(
@@ -172,6 +183,8 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             .withPolicy(new DiskTierTookTimePolicy(settings, clusterSettings, transformationFunction))
             .build();
         this.indicesService = indicesService;
+        this.settings = settings;
+        clusterSettings.addSettingsUpdateConsumer(INDICES_CACHE_DISK_TIER_ENABLED, this::toggleDiskTierEnabled);
     }
 
     @Override
@@ -466,4 +479,49 @@ long count() {
     int numRegisteredCloseListeners() { // for testing
         return registeredClosedListeners.size();
     }
+
+    /**
+     * Creates a new disk tier instance. Should only be run if the instance will actually be used!
+     * @param settings The settings to use
+     * @return A new disk tier instance
+     */
+    public EhCacheDiskCachingTier<Key, BytesReference> getDefaultDiskTier(Settings settings) {
+        String SETTING_PREFIX = "indices.request.cache";
+        long CACHE_SIZE_IN_BYTES = INDICES_CACHE_DISK_TIER_SIZE.get(settings).getBytes();
+        String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
+        long keystoreMaxWeight = INDICES_CACHE_KEYSTORE_SIZE.get(settings).getBytes();
+
+        EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
+            .setKeyType(Key.class)
+            .setValueType(BytesReference.class)
+            .setExpireAfterAccess(TimeValue.MAX_VALUE) // TODO: Is this meant to be the same as IRC expire or different?
+            .setSettings(settings)
+            .setThreadPoolAlias("ehcacheThreadpool")
+            .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+            .setStoragePath(STORAGE_PATH)
+            .setSettingPrefix(SETTING_PREFIX)
+            .setKeySerializer(new IRCKeyWriteableSerializer(this))
+            .setValueSerializer(new BytesReferenceSerializer())
+            .setKeyStoreMaxWeightInBytes(keystoreMaxWeight)
+            .build();
+        return ehcacheDiskTier;
+    }
+
+    /**
+     * A function to run when disk tier enabled setting (IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED)
+     * is enabled/disabled dynamically.
+     * This function can't go in the TieredCacheSpilloverStrategyService because of issues with generics.
+     * @param enableDiskTier the new value of the enable disk tier setting
+     */
+    private void toggleDiskTierEnabled(boolean enableDiskTier) {
+        if (enableDiskTier && tieredCacheService.getDiskCachingTier().isEmpty()) {
+            // If no disk tier existed before, create one now. Otherwise do nothing
+            EhCacheDiskCachingTier<Key, BytesReference> diskCachingTier = getDefaultDiskTier(settings);
+            tieredCacheService.addDiskTier(diskCachingTier);
+
+        } else if (!enableDiskTier && tieredCacheService.getDiskCachingTier().isPresent()){
+            // If a disk tier existed before, delete it (losing all the cached values). Otherwise do nothing
+            tieredCacheService.removeDiskTier();
+        }
+    }
 }

From 20114caede46ee0e0825482ab5214adce8c72265 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 4 Dec 2023 14:08:12 -0800
Subject: [PATCH 087/120] reverted modified github actions

---
 .github/workflows/assemble.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
index a46ee82acdcdb..6a66ac5fb5609 100644
--- a/.github/workflows/assemble.yml
+++ b/.github/workflows/assemble.yml
@@ -3,7 +3,7 @@ on: [pull_request]
 
 jobs:
   assemble:
-    if: github.repository == 'peteralfonsi/OpenSearch'
+    if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -24,8 +24,3 @@ jobs:
       - name: Run Gradle (assemble)
         run: |
           ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
-      - name: Upload artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: tiered-caching-artifact
-          path: distribution/archives/linux-tar/build/install/opensearch-3.0.0-SNAPSHOT/lib/opensearch-3.0.0-SNAPSHOT.jar

From b7fcac7b5789705103347d0f0249291ce2ec47b2 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 4 Dec 2023 14:09:21 -0800
Subject: [PATCH 088/120] reverted modified github actions

---
 .github/workflows/assemble.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
index a46ee82acdcdb..6a66ac5fb5609 100644
--- a/.github/workflows/assemble.yml
+++ b/.github/workflows/assemble.yml
@@ -3,7 +3,7 @@ on: [pull_request]
 
 jobs:
   assemble:
-    if: github.repository == 'peteralfonsi/OpenSearch'
+    if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -24,8 +24,3 @@ jobs:
       - name: Run Gradle (assemble)
         run: |
           ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
-      - name: Upload artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: tiered-caching-artifact
-          path: distribution/archives/linux-tar/build/install/opensearch-3.0.0-SNAPSHOT/lib/opensearch-3.0.0-SNAPSHOT.jar

From 5c472aed03d5f9c43cbd119db917a5104d3d3bf3 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 5 Dec 2023 07:11:27 -0800
Subject: [PATCH 089/120] Introduce CleanupStatus to keysToClean

---
 .../opensearch/indices/IndicesRequestCache.java   | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 2f8d4fe867a59..910365014117e 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -121,7 +121,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
     );
 
     private final ConcurrentMap<CleanupKey, Boolean> registeredClosedListeners = ConcurrentCollections.newConcurrentMap();
-    private final Set<CleanupKey> keysToClean = ConcurrentCollections.newConcurrentSet();
+    private final Map<CleanupKey, CleanupStatus> keysToClean = ConcurrentCollections.newConcurrentMap();
     // A map to keep track of the number of keys to be cleaned for a given ShardId and readerCacheKeyId
     private final ConcurrentMap<ShardId, ConcurrentMap<String , Integer>> diskCleanupKeyToCountMap = ConcurrentCollections.newConcurrentMap();
     private final AtomicInteger staleKeysInDiskCount = new AtomicInteger(0);
@@ -203,7 +203,7 @@ public void close() {
 
     void clear(CacheEntity entity) {
         CleanupKey cleanupKey = new CleanupKey(entity, null);
-        keysToClean.add(cleanupKey);
+        keysToClean.put(cleanupKey, new CleanupStatus());
         updateStaleKeysInDiskCount(cleanupKey);
         cleanCache();
     }
@@ -370,6 +370,11 @@ public BytesReference load(Key key) throws Exception {
         }
     }
 
+    public class CleanupStatus {
+        public boolean cleanedInHeap;
+        public boolean cleanedOnDisk;
+    }
+
     /**
      * Basic interface to make this cache testable.
      */
@@ -484,8 +489,8 @@ private CleanupKey(CacheEntity entity, String readerCacheKeyId) {
         public void onClose(IndexReader.CacheKey cacheKey) {
             Boolean remove = registeredClosedListeners.remove(this);
             if (remove != null) {
-                keysToClean.add(this);
-                updateStaleKeysInDiskCount(new CleanupKey(this.entity, this.readerCacheKeyId));
+                keysToClean.put(this, new CleanupStatus());
+                updateStaleKeysInDiskCount(this);
             }
         }
 
@@ -622,7 +627,7 @@ int numRegisteredCloseListeners() { // for testing
 
     // to be used for testing only
     void addCleanupKeyForTesting(CacheEntity entity, String readerCacheKeyId) { // for testing
-        keysToClean.add(new CleanupKey(entity, readerCacheKeyId));
+        keysToClean.put(new CleanupKey(entity, readerCacheKeyId), new CleanupStatus());
     }
 
     // to be used for testing only

From be10a165d1fe717d573a9af399a334d8ac941faa Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 5 Dec 2023 07:11:55 -0800
Subject: [PATCH 090/120] use that cleanupStatus and logic to update
 staleKeyEntries

---
 .../indices/IndicesRequestCache.java          | 94 +++++++++++++------
 1 file changed, 66 insertions(+), 28 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 910365014117e..c71f874c3e5bf 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -73,7 +73,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.Iterator;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
@@ -206,6 +206,12 @@ void clear(CacheEntity entity) {
         keysToClean.put(cleanupKey, new CleanupStatus());
         updateStaleKeysInDiskCount(cleanupKey);
         cleanCache();
+        /*
+        this would be triggered by the cache clear API call
+        we need to make sure we clean the disk cache as well
+        hence passing threshold as 0
+        */
+        cleanDiskCache(0);
     }
 
     @Override
@@ -290,7 +296,6 @@ private void updateStaleKeysInDiskCount(CleanupKey cleanupKey) {
         }
     }
 
-
     BytesReference getOrCompute(
         CacheEntity cacheEntity,
         CheckedSupplier<BytesReference, IOException> loader,
@@ -520,8 +525,33 @@ public int hashCode() {
     synchronized void cleanCache() {
         final Set<CleanupKey> currentKeysToClean = new HashSet<>();
         final Set<Object> currentFullClean = new HashSet<>();
+        /*
+        Stores the keys that need to be removed from keysToClean
+        This is done to avoid ConcurrentModificationException
+        */
+        final Set<CleanupKey> keysCleanedFromAllCaches = new HashSet<>();
+
+        for (Map.Entry<CleanupKey, CleanupStatus> entry : keysToClean.entrySet()) {
+            CleanupKey cleanupKey = entry.getKey();
+            CleanupStatus cleanupStatus = entry.getValue();
+
+            if (cleanupStatus.cleanedInHeap && cleanupStatus.cleanedOnDisk) {
+                keysCleanedFromAllCaches.add(cleanupKey);
+                continue;
+            }
+
+            if (cleanupStatus.cleanedInHeap) continue;
+
+            if (needsFullClean(cleanupKey)) {
+                currentFullClean.add(cleanupKey.entity.getCacheIdentity());
+            } else {
+                currentKeysToClean.add(cleanupKey);
+            }
+            cleanupStatus.cleanedInHeap = true;
+        }
 
-        categorizeKeysForCleanup(currentKeysToClean, currentFullClean);
+        // Remove keys that have been cleaned from all caches
+        keysToClean.keySet().removeAll(keysCleanedFromAllCaches);
 
         // Early exit if no cleanup is needed
         if (currentKeysToClean.isEmpty() && currentFullClean.isEmpty()) {
@@ -549,10 +579,37 @@ synchronized void cleanDiskCache(double diskCachesCleanThresholdPercent) {
                     return;
                 }
             }
-            Set<CleanupKey> currentKeysToClean = new HashSet<>();
-            Set<Object> currentFullClean = new HashSet<>();
+            // Categorize keys to be cleaned into currentKeysToClean and currentFullClean
+            final Set<CleanupKey> currentKeysToClean = new HashSet<>();
+            final Set<Object> currentFullClean = new HashSet<>();
+
+            /*
+            Stores the keys that need to be removed from keysToClean
+            This is done to avoid ConcurrentModificationException
+            */
+            final Set<CleanupKey> keysCleanedFromAllCaches = new HashSet<>();
+
+            for (Map.Entry<CleanupKey, CleanupStatus> entry : keysToClean.entrySet()) {
+                CleanupKey cleanupKey = entry.getKey();
+                CleanupStatus cleanupStatus = entry.getValue();
+
+                if (cleanupStatus.cleanedInHeap && cleanupStatus.cleanedOnDisk) {
+                    keysCleanedFromAllCaches.add(cleanupKey);
+                    continue;
+                }
+
+                if (cleanupStatus.cleanedOnDisk) continue;
+
+                if (needsFullClean(cleanupKey)) {
+                    currentFullClean.add(cleanupKey.entity.getCacheIdentity());
+                } else {
+                    currentKeysToClean.add(cleanupKey);
+                }
+                cleanupStatus.cleanedOnDisk = true;
+            }
 
-            categorizeKeysForCleanup(currentKeysToClean, currentFullClean);
+            // Remove keys that have been cleaned from all caches
+            keysToClean.keySet().removeAll(keysCleanedFromAllCaches);
 
             // Early exit if no cleanup is needed
             if (currentKeysToClean.isEmpty() && currentFullClean.isEmpty()) {
@@ -580,30 +637,11 @@ synchronized void cleanUpKeys(
         Set<CleanupKey> currentKeysToClean,
         Set<Object> currentFullClean
     ) {
-        Iterator<Key> iterator = cachingTier.keys().iterator();
-        while (iterator.hasNext()) {
-            Key key = iterator.next();
+        for (Key key : cachingTier.keys()) {
             CleanupKey cleanupKey = new CleanupKey(key.entity, key.readerCacheKeyId);
-            if (currentFullClean.contains(key.entity.getCacheIdentity())) {
+            if (currentFullClean.contains(key.entity.getCacheIdentity()) || currentKeysToClean.contains(cleanupKey)) {
                 cachingTier.invalidate(key);
-                currentFullClean.remove(key.entity.getCacheIdentity());
-                keysToClean.remove(cleanupKey); // since a key could be either in onHeap or disk cache.
-            } else if (currentKeysToClean.contains(cleanupKey)) {
-                cachingTier.invalidate(key);
-                currentKeysToClean.remove(cleanupKey);
-                keysToClean.remove(cleanupKey);
-            }
-        }
-    }
-
-    private void categorizeKeysForCleanup(Set<CleanupKey> currentKeysToClean, Set<Object> currentFullClean) {
-        Iterator<CleanupKey> iterator = keysToClean.iterator();
-        while (iterator.hasNext()) {
-            CleanupKey cleanupKey = iterator.next();
-            if (needsFullClean(cleanupKey)) {
-                currentFullClean.add(cleanupKey.entity.getCacheIdentity());
-            } else {
-                currentKeysToClean.add(cleanupKey);
+                staleKeysInDiskCount.decrementAndGet();
             }
         }
     }

From 4c8b2408264162d28f98eaff08ff473156f1fc6f Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 5 Dec 2023 07:12:20 -0800
Subject: [PATCH 091/120] register
 INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING to cluster settings

---
 .../java/org/opensearch/common/settings/ClusterSettings.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 0b0dfcc63c422..d7856f6cad98e 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -690,6 +690,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING,
                 IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING,
                 IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING,
+                IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING,
                 AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT,

From eeb973edfbc7f27817955f956bfaa32aa82e904c Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 5 Dec 2023 07:12:47 -0800
Subject: [PATCH 092/120] Add removal listener to update eh cache stats

---
 .../common/cache/tier/EhCacheDiskCachingTier.java      | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index f8aedbe5314de..170331d102959 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -326,6 +326,16 @@ public void onEvent(CacheEvent<? extends K, ? extends byte[]> event) {
                     assert event.getNewValue() == null;
                     break;
                 case REMOVED:
+                    this.removalListener.ifPresent(
+                        listener -> listener.onRemoval(
+                            new RemovalNotification<>(
+                                event.getKey(),
+                                valueSerializer.deserialize(event.getOldValue()),
+                                RemovalReason.INVALIDATED,
+                                TierType.DISK
+                            )
+                        )
+                    );
                     count.dec();
                     assert event.getNewValue() == null;
                     break;

From f22cdafa204893585ba21d80aa9e179eafba298d Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 5 Dec 2023 07:13:00 -0800
Subject: [PATCH 093/120] re-organize imports

---
 .../cache/tier/EhCacheDiskCachingTier.java    | 35 ++++++++-----------
 1 file changed, 14 insertions(+), 21 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index 170331d102959..39fe6d5377490 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -8,7 +8,21 @@
 
 package org.opensearch.common.cache.tier;
 
+import org.ehcache.Cache;
+import org.ehcache.CachePersistenceException;
+import org.ehcache.PersistentCacheManager;
+import org.ehcache.config.builders.CacheConfigurationBuilder;
+import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
+import org.ehcache.config.builders.CacheManagerBuilder;
+import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
+import org.ehcache.config.builders.ResourcePoolsBuilder;
+import org.ehcache.config.units.MemoryUnit;
 import org.ehcache.core.spi.service.FileBasedPersistenceContext;
+import org.ehcache.event.CacheEvent;
+import org.ehcache.event.CacheEventListener;
+import org.ehcache.event.EventType;
+import org.ehcache.expiry.ExpiryPolicy;
+import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
 import org.ehcache.spi.serialization.SerializerException;
 import org.opensearch.OpenSearchException;
 import org.opensearch.common.cache.RemovalListener;
@@ -21,35 +35,14 @@
 import org.opensearch.common.unit.TimeValue;
 
 import java.io.File;
-import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.time.Duration;
-import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Supplier;
 
-import org.ehcache.Cache;
-import org.ehcache.CachePersistenceException;
-import org.ehcache.PersistentCacheManager;
-import org.ehcache.config.builders.CacheConfigurationBuilder;
-import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
-import org.ehcache.config.builders.CacheManagerBuilder;
-import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
-import org.ehcache.config.builders.ResourcePoolsBuilder;
-import org.ehcache.config.units.MemoryUnit;
-import org.ehcache.event.CacheEvent;
-import org.ehcache.event.CacheEventListener;
-import org.ehcache.event.EventType;
-import org.ehcache.expiry.ExpiryPolicy;
-import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
-import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.unit.ByteSizeValue;
-import org.opensearch.search.query.QuerySearchResult;
-
 /**
  * An ehcache-based disk tier implementation.
  * @param <K> The key type of cache entries

From 280cb071ddade98ace91de6fe9c14c6856c6853d Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Tue, 5 Dec 2023 07:13:09 -0800
Subject: [PATCH 094/120] Add IT tests

---
 .../IndicesRequestCacheDiskTierIT.java        | 254 +++++++++++++++++-
 1 file changed, 253 insertions(+), 1 deletion(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index 4cb6eb1e3c09f..77808c2a88a68 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -45,12 +45,14 @@
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
+import java.time.Duration;
+import java.time.Instant;
+
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse;
 
 // This is a separate file from IndicesRequestCacheIT because we only want to run our test
 // on a node with a maximum request cache size that we set.
-
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class IndicesRequestCacheDiskTierIT extends OpenSearchIntegTestCase {
     public void testDiskTierStats() throws Exception {
@@ -110,7 +112,257 @@ public void testDiskTierStats() throws Exception {
         IndicesRequestCacheIT.assertCacheState(client, "index", 0,  numRequests + 3, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 2, numRequests + 1, TierType.DISK, false);
         assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, tookTimeSoFar);
+    }
+
+    public void testDiskTierInvalidationByCleanCacheAPI() throws Exception {
+        int cleanupIntervalInMillis = 10_000_000; // setting this intentionally high so that we don't get background cleanups
+        int heapSizeBytes = 9876;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(cleanupIntervalInMillis))
+                .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING.getKey(), "0%")
+                .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+                .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+        );
+        Client client = client(node);
+
+        Settings.Builder indicesSettingBuilder = Settings.builder()
+            .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+        assertAcked(
+            client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(indicesSettingBuilder).get()
+        );
+
+        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+        ensureSearchable("index");
+        SearchResponse resp;
+
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
+        int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
+        assertTrue(heapSizeBytes > requestSize);
+        // If this fails, increase heapSizeBytes! We can't adjust it after getting the size of one query
+        // as the cache size setting is not dynamic
+        int numOnDisk = 2;
+        int numRequests = heapSizeBytes / requestSize + numOnDisk;
+        RequestCacheStats requestCacheStats = client.admin()
+            .indices()
+            .prepareStats("index")
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+        for (int i = 1; i < numRequests; i++) {
+            requestCacheStats = client.admin()
+                .indices()
+                .prepareStats("index")
+                .setRequestCache(true)
+                .get()
+                .getTotal()
+                .getRequestCache();
+            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+            assertSearchResponse(resp);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
+        }
+
+        requestCacheStats = client.admin()
+            .indices()
+            .prepareStats("index")
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+
+        long entries = requestCacheStats.getEntries(TierType.DISK);
+        // make sure we have 2 entries in disk.
+        assertEquals(2, entries);
+
+        // call clear cache api
+        client.admin().indices().prepareClearCache().setIndices("index").setRequestCache(true).get();
+        // fetch the stats again
+        requestCacheStats = client.admin()
+            .indices()
+            .prepareStats("index")
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+        entries = requestCacheStats.getEntries(TierType.DISK);
+        // make sure we have 0 entries in disk.
+        assertEquals(0, entries);
+    }
+
+    // When entire disk tier is stale, test whether cache cleaner cleans up everything from disk
+    public void testDiskTierInvalidationByCacheCleanerEntireDiskTier() throws Exception {
+        int thresholdInMillis = 4_000;
+        Instant start = Instant.now();
+        int heapSizeBytes = 9876;
+        String node = internalCluster().startNode(
+        Settings.builder()
+            .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(thresholdInMillis))
+            .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING.getKey(), "1%")
+            .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+            .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+        );
+        Client client = client(node);
+
+        Settings.Builder indicesSettingBuilder = Settings.builder()
+            .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+        assertAcked(
+            client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(indicesSettingBuilder).get()
+        );
+
+        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+        ensureSearchable("index");
+        SearchResponse resp;
+
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
+        int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
+        assertTrue(heapSizeBytes > requestSize);
+
+        int numOnDisk = 2;
+        int numRequests = heapSizeBytes / requestSize + numOnDisk;
+        for (int i = 1; i < numRequests; i++) {
+            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+            assertSearchResponse(resp);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
+        }
+
+        RequestCacheStats requestCacheStats = client.admin()
+            .indices()
+            .prepareStats("index")
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+
+        // make sure we have 2 entries in disk.
+         long entries = requestCacheStats.getEntries(TierType.DISK);
+        assertEquals(2, entries);
+
+        // index a doc and force refresh so that the cache cleaner can clean the cache
+        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+        ensureSearchable("index");
+
+        // sleep for the threshold time, so that the cache cleaner can clean the cache
+        Instant end = Instant.now();
+        long elapsedTimeMillis = Duration.between(start, end).toMillis();
+        // if this test is flaky, increase the sleep time.
+        long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 1_000;
+        Thread.sleep(sleepTime);
+
+        // by now cache cleaner would have run and cleaned up stale keys
+        // fetch the stats again
+        requestCacheStats = client.admin()
+            .indices()
+            .prepareStats("index")
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+
+        // make sure we have 0 entries in disk.
+        entries = requestCacheStats.getEntries(TierType.DISK);
+        assertEquals(0, entries);
+    }
+
+    // When part of disk tier is stale, test whether cache cleaner cleans up only stale items from disk
+    public void testDiskTierInvalidationByCacheCleanerPartOfDiskTier() throws Exception {
+        int thresholdInMillis = 4_000;
+        Instant start = Instant.now();
+        int heapSizeBytes = 987;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(thresholdInMillis))
+                .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING.getKey(), "1%")
+                .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+                .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+        );
+        Client client = client(node);
+
+        Settings.Builder indicesSettingBuilder = Settings.builder()
+            .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+        assertAcked(
+            client.admin().indices().prepareCreate("index").setMapping("k", "type=text").setSettings(indicesSettingBuilder).get()
+        );
+
+        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+        ensureSearchable("index");
+        SearchResponse resp;
+
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
+        int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
+        assertTrue(heapSizeBytes > requestSize);
+
+        int numOnDisk = 2;
+        int numRequests = heapSizeBytes / requestSize + numOnDisk;
+        for (int i = 1; i < numRequests; i++) {
+            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+            assertSearchResponse(resp);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
+        }
+
+        RequestCacheStats requestCacheStats = client.admin()
+            .indices()
+            .prepareStats("index")
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+
+        // make sure we have 2 entries in disk.
+        long entries = requestCacheStats.getEntries(TierType.DISK);
+        assertEquals(2, entries);
+
+        // force refresh so that it creates stale keys in the cache for the cache cleaner to pick up.
+        flushAndRefresh("index");
+        client().prepareIndex("index").setId("1").setSource("k", "good bye");
+        ensureSearchable("index");
+
+        for (int i = 0; i < 6; i++) { // index 5 items with the new readerCacheKeyId
+            client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+        }
+
+        // fetch the stats again
+        requestCacheStats = client.admin()
+            .indices()
+            .prepareStats("index")
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+
+        entries = requestCacheStats.getEntries(TierType.DISK);
+
+        // sleep for the threshold time, so that the cache cleaner can clean the cache
+        Instant end = Instant.now();
+        long elapsedTimeMillis = Duration.between(start, end).toMillis();
+        // if this test is flaky, increase the sleep time.
+        long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 5_000;
+        Thread.sleep(sleepTime);
+
+        // fetch the stats again
+        requestCacheStats = client.admin()
+            .indices()
+            .prepareStats("index")
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
 
+        // make sure we have 5 entries in disk.
+        entries = requestCacheStats.getEntries(TierType.DISK);
+        assertEquals(5, entries);
     }
 
     private long getCacheSizeBytes(Client client, String index, TierType tierType) {

From 295ff1aa27704fd9aec0cd9c7ea727591fc8128c Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 5 Dec 2023 10:11:32 -0800
Subject: [PATCH 095/120] Added RBM size setting + UT

---
 .../cache/tier/EhCacheDiskCachingTier.java    | 12 ++++--
 .../tier/keystore/RBMIntKeyLookupStore.java   | 37 ++++++++++++++++--
 .../common/settings/ClusterSettings.java      | 10 ++++-
 .../common/settings/FeatureFlagSettings.java  |  3 +-
 .../opensearch/common/util/FeatureFlags.java  |  7 ++++
 .../indices/IndicesRequestCache.java          |  7 ++--
 .../keystore/RBMIntKeyLookupStoreTests.java   | 38 +++++++++++--------
 7 files changed, 84 insertions(+), 30 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index 639936b62fc7b..b98128640b044 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -14,6 +14,7 @@
 import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
 import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
@@ -43,6 +44,7 @@
 import org.ehcache.expiry.ExpiryPolicy;
 import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
 import org.ehcache.spi.serialization.SerializerException;
+import org.opensearch.core.common.unit.ByteSizeValue;
 
 /**
  * An ehcache-based disk tier implementation.
@@ -70,6 +72,7 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
     private final String threadPoolAlias;
 
     private final Settings settings;
+    private final ClusterSettings clusterSettings;
 
     private CounterMetric count = new CounterMetric();
 
@@ -114,6 +117,7 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
             this.threadPoolAlias = builder.threadPoolAlias;
         }
         this.settings = Objects.requireNonNull(builder.settings, "Settings objects shouldn't be null");
+        this.clusterSettings = Objects.requireNonNull(builder.clusterSettings, "ClusterSettings object shouldn't be null");
         Objects.requireNonNull(builder.settingPrefix, "Setting prefix shouldn't be null");
         this.DISK_WRITE_MINIMUM_THREADS = Setting.intSetting(builder.settingPrefix + ".tiered.disk.ehcache.min_threads", 2, 1, 5);
         this.DISK_WRITE_MAXIMUM_THREADS = Setting.intSetting(builder.settingPrefix + ".tiered.disk.ehcache.max_threads", 2, 1, 20);
@@ -128,8 +132,7 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         cacheManager = buildCacheManager();
         this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
 
-        long keystoreMaxWeight = builder.keystoreMaxWeightInBytes;
-        this.keystore = new RBMIntKeyLookupStore(keystoreMaxWeight);
+        this.keystore = new RBMIntKeyLookupStore(clusterSettings);
     }
 
     private PersistentCacheManager buildCacheManager() {
@@ -435,6 +438,7 @@ public static class Builder<K, V> {
         private String threadPoolAlias;
 
         private Settings settings;
+        private ClusterSettings clusterSettings;
 
         private String diskCacheAlias;
 
@@ -512,8 +516,8 @@ public EhCacheDiskCachingTier.Builder<K, V> setValueSerializer(Serializer<V, byt
             return this;
         }
 
-        public EhCacheDiskCachingTier.Builder<K, V> setKeyStoreMaxWeightInBytes(long weight) {
-            this.keystoreMaxWeightInBytes = weight;
+        public EhCacheDiskCachingTier.Builder<K, V> setClusterSettings(ClusterSettings clusterSettings) {
+            this.clusterSettings = clusterSettings;
             return this;
         }
 
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index bb258439de83e..d32772f07a231 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -39,6 +39,9 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.roaringbitmap.RoaringBitmap;
 
 /**
@@ -50,6 +53,13 @@
  * The store estimates its memory footprint and will stop adding more values once it reaches its memory cap.
  */
 public class RBMIntKeyLookupStore implements KeyLookupStore<Integer> {
+    public static final Setting<ByteSizeValue> INDICES_CACHE_KEYSTORE_SIZE = Setting.memorySizeSetting(
+        "index.requests.cache.tiered.disk.keystore.size",
+        "0.05%", // 5% of INDICES_CACHE_QUERY_SIZE
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
     /**
      * An enum representing modulo values for use in the keystore
      */
@@ -87,12 +97,18 @@ public int getValue() {
     // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store
     // in terms of storage impact
 
-    // Default constructor sets modulo = 2^28
-    public RBMIntKeyLookupStore(long memSizeCapInBytes) {
-        this(KeystoreModuloValue.TWO_TO_TWENTY_EIGHT, memSizeCapInBytes);
+    // Use this constructor to set the memory cap to the value in ClusterSettings
+    public RBMIntKeyLookupStore(ClusterSettings clusterSettings) {
+        this(clusterSettings.get(INDICES_CACHE_KEYSTORE_SIZE).getBytes(), clusterSettings);
+    }
+
+    // Use this constructor to specify some other memory cap
+    public RBMIntKeyLookupStore(long memSizeCapInBytes, ClusterSettings clusterSettings) {
+        this(KeystoreModuloValue.TWO_TO_TWENTY_EIGHT, memSizeCapInBytes, clusterSettings);
     }
 
-    public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBytes) {
+    // Use this constructor to specify memory cap and modulo
+    public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBytes, ClusterSettings clusterSettings) {
         this.modulo = moduloValue.getValue();
         if (modulo > 0) {
             this.modulo_bitmask = modulo - 1; // keep last log_2(modulo) bits
@@ -104,6 +120,7 @@ public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBy
         this.collidedIntCounters = new HashMap<>();
         this.removalSets = new HashMap<>();
         this.mostRecentByteEstimate = 0L;
+
     }
 
     private int transform(int value) {
@@ -363,4 +380,16 @@ CounterMetric getNumCollisionsForValue(int value) { // package private for testi
     HashSet<Integer> getRemovalSetForValue(int value) {
         return removalSets.get(transform(value));
     }
+
+    /**
+     * Run when the setting for keystore memory cap is updated.
+     * @param newMemSizeCap The new cap size.
+     */
+    protected void setMemSizeCap(ByteSizeValue newMemSizeCap) {
+        stats.memSizeCapInBytes = newMemSizeCap.getBytes();
+        mostRecentByteEstimate = getMemorySizeInBytes();
+        if (mostRecentByteEstimate > getMemorySizeCapInBytes()) {
+            stats.atCapacity.set(true);
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 5e3f4e959648e..5f1eb31aa9eab 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -80,6 +80,7 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
+import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
 import org.opensearch.common.logging.Loggers;
 import org.opensearch.common.network.NetworkModule;
 import org.opensearch.common.network.NetworkService;
@@ -692,8 +693,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT,
                 CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT,
-                IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING,
-                DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING
+                IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING
             )
         )
     );
@@ -718,6 +718,12 @@ public void apply(Settings value, Settings current, Settings previous) {
             TelemetrySettings.METRICS_PUBLISH_INTERVAL_SETTING,
             TelemetrySettings.TRACER_FEATURE_ENABLED_SETTING,
             TelemetrySettings.METRICS_FEATURE_ENABLED_SETTING
+        ),
+        List.of(FeatureFlags.TIERED_CACHING),
+        List.of(
+            RBMIntKeyLookupStore.INDICES_CACHE_KEYSTORE_SIZE,
+            IndicesRequestCache.INDICES_CACHE_DISK_STALE_KEY_THRESHOLD,
+            DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING
         )
     );
 }
diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
index 387b0c9753574..04872ab53ea02 100644
--- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
@@ -40,7 +40,8 @@ protected FeatureFlagSettings(
                 FeatureFlags.IDENTITY_SETTING,
                 FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING,
                 FeatureFlags.TELEMETRY_SETTING,
-                FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING
+                FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING,
+                FeatureFlags.TIERED_CACHING_SETTING
             )
         )
     );
diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
index 4e9b417e3433b..1516cc3719576 100644
--- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
+++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
@@ -60,6 +60,11 @@ public class FeatureFlags {
      */
     public static final String DATETIME_FORMATTER_CACHING = "opensearch.experimental.optimization.datetime_formatter_caching.enabled";
 
+    /**
+     * Gates the functionality of using a disk tier in the IndicesRequestCache.
+     */
+    public static final String TIERED_CACHING = "opensearch.experimental.feature.tiered_caching.enabled";
+
     /**
      * Should store the settings from opensearch.yml.
      */
@@ -122,4 +127,6 @@ public static boolean isEnabled(Setting<Boolean> featureFlag) {
         true,
         Property.NodeScope
     );
+
+    public static final Setting<Boolean> TIERED_CACHING_SETTING = Setting.boolSetting(TIERED_CACHING, false, Property.NodeScope);
 }
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index cb253ca6a1ed6..57ae005d531d0 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -148,19 +148,18 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         double diskTierKeystoreWeightFraction = 0.05; // Allocate 5% of the on-heap weight to the disk tier's keystore
         long keystoreMaxWeight = (long) (diskTierKeystoreWeightFraction * INDICES_CACHE_QUERY_SIZE.get(settings).getBytes());
 
-        EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>().setKeyType(
-            Key.class
-        )
+        EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
+            .setKeyType(Key.class)
             .setValueType(BytesReference.class)
             .setExpireAfterAccess(TimeValue.MAX_VALUE)
             .setSettings(settings)
+            .setClusterSettings(clusterSettings)
             .setThreadPoolAlias("ehcacheTest")
             .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
             .setStoragePath(STORAGE_PATH)
             .setSettingPrefix(SETTING_PREFIX)
             .setKeySerializer(new IRCKeyWriteableSerializer(this))
             .setValueSerializer(new BytesReferenceSerializer())
-            .setKeyStoreMaxWeightInBytes(keystoreMaxWeight)
             .build();
 
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index b1ec815980c23..7a0e91493a8d7 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -33,11 +33,14 @@
 
 import org.opensearch.common.Randomness;
 import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -47,18 +50,23 @@
 public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase {
 
     final int BYTES_IN_MB = 1048576;
+    ClusterSettings DUMMY_CLUSTER_SETTINGS = new ClusterSettings(Settings.EMPTY, Set.of(RBMIntKeyLookupStore.INDICES_CACHE_KEYSTORE_SIZE));
 
     public void testInit() {
         long memCap = 100 * BYTES_IN_MB;
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(memCap);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(memCap, DUMMY_CLUSTER_SETTINGS);
         assertEquals(0, kls.getSize());
         assertEquals(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_EIGHT.getValue(), kls.modulo);
         assertEquals(memCap, kls.getMemorySizeCapInBytes());
+
+        long clusterSettingsMemCap = DUMMY_CLUSTER_SETTINGS.get(RBMIntKeyLookupStore.INDICES_CACHE_KEYSTORE_SIZE).getBytes();
+        RBMIntKeyLookupStore clusterSettingsBasedKls = new RBMIntKeyLookupStore(DUMMY_CLUSTER_SETTINGS);
+        assertEquals(clusterSettingsMemCap, clusterSettingsBasedKls.getMemorySizeCapInBytes());
     }
 
     public void testTransformationLogic() throws Exception {
         int modulo = (int) Math.pow(2, 29);
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L, DUMMY_CLUSTER_SETTINGS);
         int offset = 3;
         for (int i = 0; i < 4; i++) { // after this we run into max value, but thats not a flaw with the class design
             int posValue = i * modulo + offset;
@@ -74,7 +82,7 @@ public void testTransformationLogic() throws Exception {
             assertTrue(kls.getInternalRepresentation(value) < modulo);
             assertTrue(kls.getInternalRepresentation(value) >= 0);
         }
-        RBMIntKeyLookupStore no_modulo_kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L);
+        RBMIntKeyLookupStore no_modulo_kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L, DUMMY_CLUSTER_SETTINGS);
         Random rand = Randomness.get();
         for (int i = 0; i < 100; i++) {
             int val = rand.nextInt();
@@ -83,8 +91,8 @@ public void testTransformationLogic() throws Exception {
     }
 
     public void testContains() throws Exception {
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
-        RBMIntKeyLookupStore noModuloKls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L, DUMMY_CLUSTER_SETTINGS);
+        RBMIntKeyLookupStore noModuloKls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L, DUMMY_CLUSTER_SETTINGS);
         for (int i = 0; i < kls.REFRESH_SIZE_EST_INTERVAL + 1000; i++) {
             // set upper bound > number of elements to trigger a size check, ensuring we test that too
             kls.add(i);
@@ -96,7 +104,7 @@ public void testContains() throws Exception {
 
     public void testAddingStatsGetters() throws Exception {
         RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX;
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L, DUMMY_CLUSTER_SETTINGS);
         kls.add(15);
         kls.add(-15);
         assertEquals(2, kls.getAddAttempts());
@@ -113,7 +121,7 @@ public void testAddingStatsGetters() throws Exception {
     public void testRegenerateStore() throws Exception {
         int numToAdd = 10000000;
         Random rand = Randomness.get();
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L, DUMMY_CLUSTER_SETTINGS);
         for (int i = 0; i < numToAdd; i++) {
             kls.add(i);
         }
@@ -131,7 +139,7 @@ public void testRegenerateStore() throws Exception {
     }
 
     public void testAddingDuplicates() throws Exception {
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0L, DUMMY_CLUSTER_SETTINGS);
         int numToAdd = 4820411;
         for (int i = 0; i < numToAdd; i++) {
             kls.add(i);
@@ -168,7 +176,7 @@ public void testMemoryCapBlocksAdd() throws Exception {
             double multiplier = RBMIntKeyLookupStore.getRBMSizeMultiplier(maxEntriesArr[i], moduloValue.getValue());
             long memSizeCapInBytes = (long) (rbmReportedSizes[i] * multiplier);
             // long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue);
-            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes);
+            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes, DUMMY_CLUSTER_SETTINGS);
             for (int j = 0; j < maxEntriesArr[i] + 5000; j++) {
                 kls.add(rand.nextInt());
             }
@@ -179,7 +187,7 @@ public void testMemoryCapBlocksAdd() throws Exception {
     public void testConcurrency() throws Exception {
         Random rand = Randomness.get();
         for (int j = 0; j < 5; j++) { // test with different numbers of threads
-            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
+            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L, DUMMY_CLUSTER_SETTINGS);
             int numThreads = rand.nextInt(50) + 1;
             ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(numThreads);
             // In this test we want to add the first 200K numbers and check they're all correctly there.
@@ -241,7 +249,7 @@ public void testConcurrency() throws Exception {
     public void testRemoveNoCollisions() throws Exception {
         long memCap = 100L * BYTES_IN_MB;
         int numToAdd = 195000;
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, memCap);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, memCap, DUMMY_CLUSTER_SETTINGS);
         // there should be no collisions for sequential positive numbers up to modulo
         for (int i = 0; i < numToAdd; i++) {
             kls.add(i);
@@ -257,7 +265,7 @@ public void testRemoveNoCollisions() throws Exception {
     public void testRemoveWithCollisions() throws Exception {
         int modulo = (int) Math.pow(2, 26);
         long memCap = 100L * BYTES_IN_MB;
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX, memCap);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX, memCap, DUMMY_CLUSTER_SETTINGS);
         for (int i = 0; i < 10; i++) {
             kls.add(i);
             if (i % 2 == 1) {
@@ -300,7 +308,7 @@ public void testRemoveWithCollisions() throws Exception {
     }
 
     public void testNullInputs() throws Exception {
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L, DUMMY_CLUSTER_SETTINGS);
         assertFalse(kls.add(null));
         assertFalse(kls.contains(null));
         assertEquals(0, (int) kls.getInternalRepresentation(null));
@@ -315,7 +323,7 @@ public void testNullInputs() throws Exception {
     public void testRemovalLogic() throws Exception {
         RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX;
         int modulo = moduloValue.getValue();
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L, DUMMY_CLUSTER_SETTINGS);
 
         // Test standard sequence: add K1, K2, K3 which all transform to C, then:
         // Remove K3
@@ -381,7 +389,7 @@ public void testRemovalLogic() throws Exception {
     public void testRemovalLogicWithHashCollision() throws Exception {
         RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX;
         int modulo = moduloValue.getValue();
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L, DUMMY_CLUSTER_SETTINGS);
 
         // Test adding K1 twice (maybe two keys hash to K1), then removing it twice.
         // We expect it to be unable to remove the last one, but there should be no false negatives.

From 163d7fef7fc58a833cb83133bae5171cc88890c1 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 5 Dec 2023 10:30:21 -0800
Subject: [PATCH 096/120] added UT for setting switch

---
 .../IndicesRequestCacheDiskTierIT.java        | 23 +++++++++++++++++++
 .../tier/keystore/RBMIntKeyLookupStore.java   |  2 +-
 .../common/settings/ClusterSettings.java      |  2 +-
 .../keystore/RBMIntKeyLookupStoreTests.java   | 17 ++++++++++++++
 4 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index 18a37d4d43b42..0771ff07e3870 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -114,6 +114,29 @@ public void testDiskTierStats() throws Exception {
 
     }
 
+    public void testRBMSizeSetting() throws Exception {
+        int heapSizeBytes = 0;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+                .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of
+            // took time
+        );
+        Client client = client(node);
+
+        Settings.Builder indicesSettingBuilder = Settings.builder()
+            .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+        assertAcked(
+            client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(indicesSettingBuilder).get()
+        );
+        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+        ensureSearchable("index");
+        SearchResponse resp;
+    }
+
     private long getCacheSizeBytes(Client client, String index, TierType tierType) {
         RequestCacheStats requestCacheStats = client.admin()
             .indices()
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index d32772f07a231..343aeb4d126b8 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -92,7 +92,7 @@ public int getValue() {
     protected final Lock readLock = lock.readLock();
     protected final Lock writeLock = lock.writeLock();
     private long mostRecentByteEstimate;
-    protected final int REFRESH_SIZE_EST_INTERVAL = 10000;
+    protected static final int REFRESH_SIZE_EST_INTERVAL = 10000;
     // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms,
     // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store
     // in terms of storage impact
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 5f1eb31aa9eab..fb695d5c76b5a 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -722,7 +722,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         List.of(FeatureFlags.TIERED_CACHING),
         List.of(
             RBMIntKeyLookupStore.INDICES_CACHE_KEYSTORE_SIZE,
-            IndicesRequestCache.INDICES_CACHE_DISK_STALE_KEY_THRESHOLD,
+            //IndicesRequestCache.INDICES_CACHE_DISK_STALE_KEY_THRESHOLD,
             DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING
         )
     );
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index 7a0e91493a8d7..433de5ff6fd5e 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -35,6 +35,8 @@
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.ArrayList;
@@ -421,4 +423,19 @@ public void testRemovalLogicWithHashCollision() throws Exception {
         assertTrue(kls.contains(k1));
         assertTrue(kls.contains(k2));
     }
+
+    public void testSetMemSizeCap() throws Exception {
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0L, DUMMY_CLUSTER_SETTINGS);
+        Random rand = Randomness.get();
+        for (int i = 0; i < RBMIntKeyLookupStore.REFRESH_SIZE_EST_INTERVAL * 3; i++) {
+            kls.add(rand.nextInt());
+        }
+        long memSize = kls.getMemorySizeInBytes();
+        assertEquals(0, kls.getMemorySizeCapInBytes());
+        kls.setMemSizeCap(new ByteSizeValue(memSize/2, ByteSizeUnit.BYTES));
+        // check the keystore is now full and has its lower cap
+        assertTrue(kls.isFull());
+        assertEquals(memSize/2, kls.getMemorySizeCapInBytes());
+        assertFalse(kls.add(rand.nextInt())); 
+    }
 }

From 29d31b486463ea3efad96ab526ff9a283dff68b9 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 5 Dec 2023 12:16:55 -0800
Subject: [PATCH 097/120] added skeleton for stale key threshold setting

---
 .../common/cache/tier/EhCacheDiskCachingTier.java | 15 ++++++++++++++-
 .../common/settings/ClusterSettings.java          |  3 ++-
 .../tier/keystore/RBMIntKeyLookupStoreTests.java  |  2 +-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index b98128640b044..377a3e718b6ca 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -17,6 +17,7 @@
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.unit.TimeValue;
 
 import java.io.File;
@@ -44,7 +45,7 @@
 import org.ehcache.expiry.ExpiryPolicy;
 import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
 import org.ehcache.spi.serialization.SerializerException;
-import org.opensearch.core.common.unit.ByteSizeValue;
+
 
 /**
  * An ehcache-based disk tier implementation.
@@ -52,6 +53,12 @@
  * @param <V> The value type of cache entries
  */
 public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
+    public static final Setting<Double> INDICES_CACHE_DISK_STALE_KEY_THRESHOLD = Setting.doubleSetting(
+        "index.requests.cache.tiered.disk.stale_cleanup_threshold",
+        0.5,
+        Property.Dynamic,
+        Property.NodeScope
+    );
 
     // A Cache manager can create many caches.
     private static PersistentCacheManager cacheManager = null;
@@ -133,6 +140,7 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
 
         this.keystore = new RBMIntKeyLookupStore(clusterSettings);
+        clusterSettings.addSettingsUpdateConsumer(INDICES_CACHE_DISK_STALE_KEY_THRESHOLD, this::setStaleKeyThreshold);
     }
 
     private PersistentCacheManager buildCacheManager() {
@@ -281,6 +289,11 @@ public void close() {
         } catch (NullPointerException ignored) {} // Another test node has already destroyed the cache manager
     }
 
+    private void setStaleKeyThreshold(double newThreshold) {
+        return;
+        //TODO: Fill in once Kiran's code is merged
+    }
+
     /**
      * Wrapper over Ehcache original listener to listen to desired events and notify desired subscribers.
      * @param <K> Type of key
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index fb695d5c76b5a..1b800004e02e4 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -80,6 +80,7 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
+import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
 import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
 import org.opensearch.common.logging.Loggers;
 import org.opensearch.common.network.NetworkModule;
@@ -722,7 +723,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         List.of(FeatureFlags.TIERED_CACHING),
         List.of(
             RBMIntKeyLookupStore.INDICES_CACHE_KEYSTORE_SIZE,
-            //IndicesRequestCache.INDICES_CACHE_DISK_STALE_KEY_THRESHOLD,
+            EhCacheDiskCachingTier.INDICES_CACHE_DISK_STALE_KEY_THRESHOLD,
             DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING
         )
     );
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index 433de5ff6fd5e..31aec942b2c9d 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -436,6 +436,6 @@ public void testSetMemSizeCap() throws Exception {
         // check the keystore is now full and has its lower cap
         assertTrue(kls.isFull());
         assertEquals(memSize/2, kls.getMemorySizeCapInBytes());
-        assertFalse(kls.add(rand.nextInt())); 
+        assertFalse(kls.add(rand.nextInt()));
     }
 }

From 856160eb40a9342f6278683d57d8fcaf9410ded1 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 5 Dec 2023 12:51:13 -0800
Subject: [PATCH 098/120] Added commented-out lines to opensearch.yml

---
 distribution/src/config/opensearch.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml
index b7ab2e1c2309b..e87512977e394 100644
--- a/distribution/src/config/opensearch.yml
+++ b/distribution/src/config/opensearch.yml
@@ -127,3 +127,12 @@ ${path.logs}
 # Once there is no observed impact on performance, this feature flag can be removed.
 #
 #opensearch.experimental.optimization.datetime_formatter_caching.enabled: false
+#
+#
+# Gates the tiered caching feature. This feature adds a disk tier to the IndicesRequestCache.
+# opensearch.experimental.feature.tiered_caching.enabled: false
+#
+# Settings related to tiered caching:
+# index.requests.cache.tiered.disk.keystore.size: 0.05%
+# index.requests.cache.tiered.disk.stale_cleanup_threshold: 0.5
+#

From 4bdf459338d0e87e8b0f6db1b96358e030906682 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 5 Dec 2023 16:15:33 -0800
Subject: [PATCH 099/120] added feature flag

---
 .../indices/IndicesRequestCacheIT.java        |  5 +-
 .../cache/tier/DiskTierTookTimePolicy.java    |  2 +-
 .../indices/IndicesRequestCache.java          | 56 +++++++++----------
 3 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index e85cac7cf157c..3129352f33973 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -85,7 +85,10 @@ public static Collection<Object[]> parameters() {
 
     @Override
     protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
+        return Settings.builder().put(super.featureFlagSettings())
+            .put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true")
+            .put(FeatureFlags.TIERED_CACHING, "true")
+            .build();
     }
 
     // One of the primary purposes of the query cache is to cache aggs results
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
index d6c3b894a974d..5e6f3b2d4be26 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
@@ -29,7 +29,7 @@
  */
 public class DiskTierTookTimePolicy implements CacheTierPolicy<BytesReference> {
     public static final Setting<TimeValue> DISK_TOOKTIME_THRESHOLD_SETTING = Setting.positiveTimeSetting(
-        "index.requests.cache.disk.tooktime.threshold",
+        "indices.requests.cache.disk.tooktime.threshold",
         new TimeValue(10),
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 57ae005d531d0..03faacbde1a70 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -58,6 +58,7 @@
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.util.concurrent.ConcurrentCollections;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -140,36 +141,35 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
                 throw new RuntimeException(e);
             }
         };
-        // enabling this for testing purposes. Remove/tweak!!
-        long CACHE_SIZE_IN_BYTES = 1000000L;
-        String SETTING_PREFIX = "indices.request.cache";
-        String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
-
-        double diskTierKeystoreWeightFraction = 0.05; // Allocate 5% of the on-heap weight to the disk tier's keystore
-        long keystoreMaxWeight = (long) (diskTierKeystoreWeightFraction * INDICES_CACHE_QUERY_SIZE.get(settings).getBytes());
-
-        EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
-            .setKeyType(Key.class)
-            .setValueType(BytesReference.class)
-            .setExpireAfterAccess(TimeValue.MAX_VALUE)
-            .setSettings(settings)
-            .setClusterSettings(clusterSettings)
-            .setThreadPoolAlias("ehcacheTest")
-            .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
-            .setStoragePath(STORAGE_PATH)
-            .setSettingPrefix(SETTING_PREFIX)
-            .setKeySerializer(new IRCKeyWriteableSerializer(this))
-            .setValueSerializer(new BytesReferenceSerializer())
-            .build();
 
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
-        tieredCacheService = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>().setOnHeapCachingTier(
-            openSearchOnHeapCache
-        )
-            .setOnDiskCachingTier(ehcacheDiskTier)
-            .setTieredCacheEventListener(this)
-            .withPolicy(new DiskTierTookTimePolicy(settings, clusterSettings, transformationFunction))
-            .build();
+        TieredCacheSpilloverStrategyService.Builder<Key, BytesReference> tieredCacheServiceBuilder = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>()
+            .setOnHeapCachingTier(openSearchOnHeapCache)
+            .setTieredCacheEventListener(this);
+
+        if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
+            // enabling this for testing purposes. Remove/tweak!!
+            long CACHE_SIZE_IN_BYTES = 1000000L;
+            String SETTING_PREFIX = "indices.request.cache";
+            String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
+
+            EhCacheDiskCachingTier<Key, BytesReference> diskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
+                .setKeyType(Key.class)
+                .setValueType(BytesReference.class)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setSettings(settings)
+                .setClusterSettings(clusterSettings)
+                .setThreadPoolAlias("ehcacheTest")
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setStoragePath(STORAGE_PATH)
+                .setSettingPrefix(SETTING_PREFIX)
+                .setKeySerializer(new IRCKeyWriteableSerializer(this))
+                .setValueSerializer(new BytesReferenceSerializer())
+                .build();
+
+            tieredCacheServiceBuilder.setOnDiskCachingTier(diskTier);
+        }
+        tieredCacheService = tieredCacheServiceBuilder.build();
         this.indicesService = indicesService;
     }
 

From 347a14cd3a9955d0533b97f6b7367cc56ba9147d Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 7 Dec 2023 10:04:26 -0800
Subject: [PATCH 100/120] added ehcache cluster settings

---
 .../cache/tier/EhCacheDiskCachingTier.java    | 66 +++++++++++--------
 .../common/settings/ClusterSettings.java      |  6 +-
 2 files changed, 44 insertions(+), 28 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index 377a3e718b6ca..ff768c4e30027 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -53,13 +53,45 @@
  * @param <V> The value type of cache entries
  */
 public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
-    public static final Setting<Double> INDICES_CACHE_DISK_STALE_KEY_THRESHOLD = Setting.doubleSetting(
-        "index.requests.cache.tiered.disk.stale_cleanup_threshold",
+    public static final Setting<Double> REQUEST_CACHE_DISK_STALE_KEY_THRESHOLD = Setting.doubleSetting(
+        "indices.requests.cache.tiered.disk.stale_cleanup_threshold",
         0.5,
         Property.Dynamic,
         Property.NodeScope
     );
 
+    // Ehcache disk write minimum threads for its pool
+    public static final Setting<Integer> REQUEST_CACHE_DISK_MIN_THREADS = Setting.intSetting(
+        "indices.requests.cache.tiered.disk.ehcache.min_threads",
+        2, 1, 5,
+        Property.NodeScope
+    );
+
+    // Ehcache disk write maximum threads for its pool
+    public static final Setting<Integer> REQUEST_CACHE_DISK_MAX_THREADS = Setting.intSetting(
+        "indices.requests.cache.tiered.disk.ehcache.max_threads",
+        2, 1, 20,
+        Property.NodeScope
+    );
+
+    // Not be to confused with number of disk segments, this is different. Defines
+    // distinct write queues created for disk store where a group of segments share a write queue. This is
+    // implemented with ehcache using a partitioned thread pool exectutor By default all segments share a single write
+    // queue ie write concurrency is 1. Check OffHeapDiskStoreConfiguration and DiskWriteThreadPool.
+    public static final Setting<Integer> REQUEST_CACHE_DISK_WRITE_CONCURRENCY = Setting.intSetting(
+        "indices.requests.cache.tiered.disk.ehcache.write_concurrency",
+        2, 1, 3,
+        Property.NodeScope
+    );
+
+    // Defines how many segments the disk cache is separated into. Higher number achieves greater concurrency but
+    // will hold that many file pointers.
+    public static final Setting<Integer> REQUEST_CACHE_DISK_SEGMENTS = Setting.intSetting(
+        "indices.requests.cache.tiered.disk.ehcache.segments",
+        16, 1, 32,
+        Property.NodeScope
+    );
+
     // A Cache manager can create many caches.
     private static PersistentCacheManager cacheManager = null;
 
@@ -89,21 +121,6 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
 
     private final static int MINIMUM_MAX_SIZE_IN_BYTES = 1024 * 100; // 100KB
 
-    // Ehcache disk write minimum threads for its pool
-    public final Setting<Integer> DISK_WRITE_MINIMUM_THREADS;
-
-    // Ehcache disk write maximum threads for its pool
-    public final Setting<Integer> DISK_WRITE_MAXIMUM_THREADS;
-
-    // Not be to confused with number of disk segments, this is different. Defines
-    // distinct write queues created for disk store where a group of segments share a write queue. This is
-    // implemented with ehcache using a partitioned thread pool exectutor By default all segments share a single write
-    // queue ie write concurrency is 1. Check OffHeapDiskStoreConfiguration and DiskWriteThreadPool.
-    public final Setting<Integer> DISK_WRITE_CONCURRENCY;
-
-    // Defines how many segments the disk cache is separated into. Higher number achieves greater concurrency but
-    // will hold that many file pointers.
-    public final Setting<Integer> DISK_SEGMENTS;
     private final RBMIntKeyLookupStore keystore;
 
     private final Serializer<K, byte[]> keySerializer;
@@ -126,12 +143,7 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.settings = Objects.requireNonNull(builder.settings, "Settings objects shouldn't be null");
         this.clusterSettings = Objects.requireNonNull(builder.clusterSettings, "ClusterSettings object shouldn't be null");
         Objects.requireNonNull(builder.settingPrefix, "Setting prefix shouldn't be null");
-        this.DISK_WRITE_MINIMUM_THREADS = Setting.intSetting(builder.settingPrefix + ".tiered.disk.ehcache.min_threads", 2, 1, 5);
-        this.DISK_WRITE_MAXIMUM_THREADS = Setting.intSetting(builder.settingPrefix + ".tiered.disk.ehcache.max_threads", 2, 1, 20);
-        // Default value is 1 within EhCache.
-        this.DISK_WRITE_CONCURRENCY = Setting.intSetting(builder.settingPrefix + ".tiered.disk.ehcache.concurrency", 2, 1, 3);
-        // Default value is 16 within Ehcache.
-        this.DISK_SEGMENTS = Setting.intSetting(builder.settingPrefix + ".ehcache.disk.segments", 16, 1, 32);
+
 
         // In test cases, there might be leftover cache managers and caches hanging around, from nodes created in the test case setup
         // Destroy them before recreating them
@@ -140,7 +152,7 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
 
         this.keystore = new RBMIntKeyLookupStore(clusterSettings);
-        clusterSettings.addSettingsUpdateConsumer(INDICES_CACHE_DISK_STALE_KEY_THRESHOLD, this::setStaleKeyThreshold);
+        clusterSettings.addSettingsUpdateConsumer(REQUEST_CACHE_DISK_STALE_KEY_THRESHOLD, this::setStaleKeyThreshold);
     }
 
     private PersistentCacheManager buildCacheManager() {
@@ -151,7 +163,7 @@ private PersistentCacheManager buildCacheManager() {
                 PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
                     .defaultPool(THREAD_POOL_ALIAS_PREFIX + "Default", 1, 3) // Default pool used for other tasks like
                     // event listeners
-                    .pool(this.threadPoolAlias, DISK_WRITE_MINIMUM_THREADS.get(settings), DISK_WRITE_MAXIMUM_THREADS.get(settings))
+                    .pool(this.threadPoolAlias, clusterSettings.get(REQUEST_CACHE_DISK_MIN_THREADS), clusterSettings.get(REQUEST_CACHE_DISK_MAX_THREADS))
                     .build()
             )
             .build(true);
@@ -184,8 +196,8 @@ public Duration getExpiryForUpdate(K key, Supplier<? extends byte[]> oldValue, b
                 .withService(
                     new OffHeapDiskStoreConfiguration(
                         this.threadPoolAlias,
-                        DISK_WRITE_CONCURRENCY.get(settings),
-                        DISK_SEGMENTS.get(settings)
+                        clusterSettings.get(REQUEST_CACHE_DISK_WRITE_CONCURRENCY),
+                        clusterSettings.get(REQUEST_CACHE_DISK_SEGMENTS)
                     )
                 )
                 .withKeySerializer(new KeySerializerWrapper<K>(keySerializer))
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 1b800004e02e4..4f09d04557178 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -723,7 +723,11 @@ public void apply(Settings value, Settings current, Settings previous) {
         List.of(FeatureFlags.TIERED_CACHING),
         List.of(
             RBMIntKeyLookupStore.INDICES_CACHE_KEYSTORE_SIZE,
-            EhCacheDiskCachingTier.INDICES_CACHE_DISK_STALE_KEY_THRESHOLD,
+            EhCacheDiskCachingTier.REQUEST_CACHE_DISK_STALE_KEY_THRESHOLD,
+            EhCacheDiskCachingTier.REQUEST_CACHE_DISK_MIN_THREADS,
+            EhCacheDiskCachingTier.REQUEST_CACHE_DISK_MAX_THREADS,
+            EhCacheDiskCachingTier.REQUEST_CACHE_DISK_WRITE_CONCURRENCY,
+            EhCacheDiskCachingTier.REQUEST_CACHE_DISK_SEGMENTS,
             DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING
         )
     );

From 64c339ab584b82e20e9d4fb7ae99d821d398e97e Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 7 Dec 2023 11:47:41 -0800
Subject: [PATCH 101/120] added feature flag setting to disk tier IT

---
 .../opensearch/indices/IndicesRequestCacheDiskTierIT.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index 0771ff07e3870..cf5cca577dd19 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -39,6 +39,7 @@
 import org.opensearch.common.cache.tier.TierType;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.cache.request.ShardRequestCache;
@@ -53,6 +54,13 @@
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class IndicesRequestCacheDiskTierIT extends OpenSearchIntegTestCase {
+    @Override
+    protected Settings featureFlagSettings() {
+        return Settings.builder().put(super.featureFlagSettings())
+            .put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true")
+            .put(FeatureFlags.TIERED_CACHING, "true")
+            .build();
+    }
     public void testDiskTierStats() throws Exception {
         int heapSizeBytes = 9876;
         String node = internalCluster().startNode(

From 90c0f9a850e9e9632c49deeb04ed1a080b2d0383 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Thu, 7 Dec 2023 12:56:50 -0800
Subject: [PATCH 102/120] added null checks to key and dummy string serializers

---
 .../java/org/opensearch/indices/IRCKeyWriteableSerializer.java | 3 +++
 .../common/cache/tier/EhCacheDiskCachingTierTests.java         | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java b/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
index 92e874518df54..55d770804431f 100644
--- a/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
+++ b/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
@@ -41,6 +41,9 @@ public byte[] serialize(IndicesRequestCache.Key object) {
 
     @Override
     public IndicesRequestCache.Key deserialize(byte[] bytes) {
+        if (bytes == null) {
+            return null;
+        }
         try {
             BytesStreamInput is = new BytesStreamInput(bytes, 0, bytes.length);
             return irc.new Key(is);
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
index 139e82b274d34..eb0ecad79a045 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
@@ -325,6 +325,9 @@ public byte[] serialize(String object) {
 
         @Override
         public String deserialize(byte[] bytes) {
+            if (bytes == null) {
+                return null;
+            }
             return new String(bytes, charset);
         }
 

From 47453e1f32d41b2b8e3820eb81d4e51708b1db09 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Fri, 8 Dec 2023 09:39:35 -0800
Subject: [PATCH 103/120] Integrated misc changes from other branch

---
 distribution/src/config/opensearch.yml        |  4 ++
 .../IndicesRequestCacheDiskTierIT.java        | 28 ++--------
 .../indices/IndicesRequestCacheIT.java        |  3 +-
 .../cache/tier/EhCacheDiskCachingTier.java    | 28 ++++++----
 .../tier/keystore/RBMIntKeyLookupStore.java   | 18 ++++---
 .../cache/request/RequestCacheStats.java      | 21 ++++----
 .../indices/IndicesRequestCache.java          | 12 ++---
 .../tier/EhCacheDiskCachingTierTests.java     | 52 +++++++-----------
 .../keystore/RBMIntKeyLookupStoreTests.java   | 54 +++++++++++++++----
 9 files changed, 118 insertions(+), 102 deletions(-)

diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml
index e87512977e394..b0f801fccb155 100644
--- a/distribution/src/config/opensearch.yml
+++ b/distribution/src/config/opensearch.yml
@@ -135,4 +135,8 @@ ${path.logs}
 # Settings related to tiered caching:
 # index.requests.cache.tiered.disk.keystore.size: 0.05%
 # index.requests.cache.tiered.disk.stale_cleanup_threshold: 0.5
+# indices.requests.cache.tiered.disk.ehcache.min_threads: 2
+# indices.requests.cache.tiered.disk.ehcache.max_threads: 2
+# indices.requests.cache.tiered.disk.ehcache.write_concurrency: 2
+# indices.requests.cache.tiered.disk.ehcache.segments: 16
 #
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index cf5cca577dd19..f14f315b236a5 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -56,11 +56,13 @@
 public class IndicesRequestCacheDiskTierIT extends OpenSearchIntegTestCase {
     @Override
     protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings())
+        return Settings.builder()
+            .put(super.featureFlagSettings())
             .put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true")
             .put(FeatureFlags.TIERED_CACHING, "true")
             .build();
     }
+
     public void testDiskTierStats() throws Exception {
         int heapSizeBytes = 9876;
         String node = internalCluster().startNode(
@@ -119,30 +121,6 @@ public void testDiskTierStats() throws Exception {
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 3, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 2, numRequests + 1, TierType.DISK, false);
         assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, tookTimeSoFar);
-
-    }
-
-    public void testRBMSizeSetting() throws Exception {
-        int heapSizeBytes = 0;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
-                .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of
-            // took time
-        );
-        Client client = client(node);
-
-        Settings.Builder indicesSettingBuilder = Settings.builder()
-            .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
-
-        assertAcked(
-            client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(indicesSettingBuilder).get()
-        );
-        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
-        ensureSearchable("index");
-        SearchResponse resp;
     }
 
     private long getCacheSizeBytes(Client client, String index, TierType tierType) {
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index 3129352f33973..f07f667f71d5b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -85,7 +85,8 @@ public static Collection<Object[]> parameters() {
 
     @Override
     protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings())
+        return Settings.builder()
+            .put(super.featureFlagSettings())
             .put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true")
             .put(FeatureFlags.TIERED_CACHING, "true")
             .build();
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index ff768c4e30027..a330d99d5b589 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -16,8 +16,8 @@
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
-import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.Setting.Property;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 
 import java.io.File;
@@ -46,7 +46,6 @@
 import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
 import org.ehcache.spi.serialization.SerializerException;
 
-
 /**
  * An ehcache-based disk tier implementation.
  * @param <K> The key type of cache entries
@@ -63,14 +62,18 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
     // Ehcache disk write minimum threads for its pool
     public static final Setting<Integer> REQUEST_CACHE_DISK_MIN_THREADS = Setting.intSetting(
         "indices.requests.cache.tiered.disk.ehcache.min_threads",
-        2, 1, 5,
+        2,
+        1,
+        5,
         Property.NodeScope
     );
 
     // Ehcache disk write maximum threads for its pool
     public static final Setting<Integer> REQUEST_CACHE_DISK_MAX_THREADS = Setting.intSetting(
         "indices.requests.cache.tiered.disk.ehcache.max_threads",
-        2, 1, 20,
+        2,
+        1,
+        20,
         Property.NodeScope
     );
 
@@ -80,7 +83,9 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
     // queue ie write concurrency is 1. Check OffHeapDiskStoreConfiguration and DiskWriteThreadPool.
     public static final Setting<Integer> REQUEST_CACHE_DISK_WRITE_CONCURRENCY = Setting.intSetting(
         "indices.requests.cache.tiered.disk.ehcache.write_concurrency",
-        2, 1, 3,
+        2,
+        1,
+        3,
         Property.NodeScope
     );
 
@@ -88,7 +93,9 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
     // will hold that many file pointers.
     public static final Setting<Integer> REQUEST_CACHE_DISK_SEGMENTS = Setting.intSetting(
         "indices.requests.cache.tiered.disk.ehcache.segments",
-        16, 1, 32,
+        16,
+        1,
+        32,
         Property.NodeScope
     );
 
@@ -144,7 +151,6 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.clusterSettings = Objects.requireNonNull(builder.clusterSettings, "ClusterSettings object shouldn't be null");
         Objects.requireNonNull(builder.settingPrefix, "Setting prefix shouldn't be null");
 
-
         // In test cases, there might be leftover cache managers and caches hanging around, from nodes created in the test case setup
         // Destroy them before recreating them
         close();
@@ -163,7 +169,11 @@ private PersistentCacheManager buildCacheManager() {
                 PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
                     .defaultPool(THREAD_POOL_ALIAS_PREFIX + "Default", 1, 3) // Default pool used for other tasks like
                     // event listeners
-                    .pool(this.threadPoolAlias, clusterSettings.get(REQUEST_CACHE_DISK_MIN_THREADS), clusterSettings.get(REQUEST_CACHE_DISK_MAX_THREADS))
+                    .pool(
+                        this.threadPoolAlias,
+                        clusterSettings.get(REQUEST_CACHE_DISK_MIN_THREADS),
+                        clusterSettings.get(REQUEST_CACHE_DISK_MAX_THREADS)
+                    )
                     .build()
             )
             .build(true);
@@ -303,7 +313,7 @@ public void close() {
 
     private void setStaleKeyThreshold(double newThreshold) {
         return;
-        //TODO: Fill in once Kiran's code is merged
+        // TODO: Fill in once Kiran's code is merged
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index 343aeb4d126b8..5027813604042 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -33,15 +33,15 @@
 package org.opensearch.common.cache.tier.keystore;
 
 import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.core.common.unit.ByteSizeValue;
 
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.common.settings.Setting;
-import org.opensearch.core.common.unit.ByteSizeValue;
 import org.roaringbitmap.RoaringBitmap;
 
 /**
@@ -54,7 +54,7 @@
  */
 public class RBMIntKeyLookupStore implements KeyLookupStore<Integer> {
     public static final Setting<ByteSizeValue> INDICES_CACHE_KEYSTORE_SIZE = Setting.memorySizeSetting(
-        "index.requests.cache.tiered.disk.keystore.size",
+        "indices.requests.cache.tiered.disk.keystore.size",
         "0.05%", // 5% of INDICES_CACHE_QUERY_SIZE
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
@@ -92,7 +92,7 @@ public int getValue() {
     protected final Lock readLock = lock.readLock();
     protected final Lock writeLock = lock.writeLock();
     private long mostRecentByteEstimate;
-    protected static final int REFRESH_SIZE_EST_INTERVAL = 10000;
+    public static final int REFRESH_SIZE_EST_INTERVAL = 10000;
     // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms,
     // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store
     // in terms of storage impact
@@ -120,7 +120,7 @@ public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBy
         this.collidedIntCounters = new HashMap<>();
         this.removalSets = new HashMap<>();
         this.mostRecentByteEstimate = 0L;
-
+        clusterSettings.addSettingsUpdateConsumer(INDICES_CACHE_KEYSTORE_SIZE, this::setMemSizeCap);
     }
 
     private int transform(int value) {
@@ -147,7 +147,7 @@ public boolean add(Integer value) {
         stats.numAddAttempts.inc();
 
         if (getSize() % REFRESH_SIZE_EST_INTERVAL == 0) {
-            mostRecentByteEstimate = getMemorySizeInBytes();
+            mostRecentByteEstimate = computeMemorySizeInBytes();
         }
         if (getMemorySizeCapInBytes() > 0 && mostRecentByteEstimate > getMemorySizeCapInBytes()) {
             stats.atCapacity.set(true);
@@ -321,6 +321,10 @@ static double getRBMSizeMultiplier(int numEntries, int modulo) {
 
     @Override
     public long getMemorySizeInBytes() {
+        return mostRecentByteEstimate;
+    }
+
+    private long computeMemorySizeInBytes() {
         double multiplier = getRBMSizeMultiplier((int) stats.size.count(), modulo);
         return (long) (rbm.getSizeInBytes() * multiplier);
     }
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index 4836479133b67..a5ab4e8354aaa 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -35,6 +35,7 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
 import org.opensearch.common.cache.tier.TierType;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -193,18 +194,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         // write on-heap stats outside of tiers object
         getTierStats(TierType.ON_HEAP).toXContent(builder, params);
         getTierSpecificStats(TierType.ON_HEAP).toXContent(builder, params);
-        builder.startObject(Fields.TIERS);
-        for (TierType tierType : TierType.values()) { // fixed order
-            if (tierType != TierType.ON_HEAP) {
-                String tier = tierType.getStringValue();
-                builder.startObject(tier);
-                defaultStatsMap.get(tier).toXContent(builder, params);
-                tierSpecificStatsMap.get(tier).toXContent(builder, params);
-                builder.endObject();
+        if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
+            builder.startObject(Fields.TIERS);
+            for (TierType tierType : TierType.values()) { // fixed order
+                if (tierType != TierType.ON_HEAP) {
+                    String tier = tierType.getStringValue();
+                    builder.startObject(tier);
+                    defaultStatsMap.get(tier).toXContent(builder, params);
+                    tierSpecificStatsMap.get(tier).toXContent(builder, params);
+                    builder.endObject();
+                }
             }
+            builder.endObject();
         }
         builder.endObject();
-        builder.endObject();
         return builder;
     }
 
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 03faacbde1a70..750f114ccee15 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -43,7 +43,6 @@
 import org.opensearch.common.cache.tier.BytesReferenceSerializer;
 import org.opensearch.common.cache.tier.CachePolicyInfoWrapper;
 import org.opensearch.common.cache.tier.CacheValue;
-import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
 import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
 import org.opensearch.common.cache.tier.OnHeapCachingTier;
 import org.opensearch.common.cache.tier.OpenSearchOnHeapCache;
@@ -143,9 +142,9 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         };
 
         // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
-        TieredCacheSpilloverStrategyService.Builder<Key, BytesReference> tieredCacheServiceBuilder = new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>()
-            .setOnHeapCachingTier(openSearchOnHeapCache)
-            .setTieredCacheEventListener(this);
+        TieredCacheSpilloverStrategyService.Builder<Key, BytesReference> tieredCacheServiceBuilder =
+            new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>().setOnHeapCachingTier(openSearchOnHeapCache)
+                .setTieredCacheEventListener(this);
 
         if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
             // enabling this for testing purposes. Remove/tweak!!
@@ -153,8 +152,9 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             String SETTING_PREFIX = "indices.request.cache";
             String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
 
-            EhCacheDiskCachingTier<Key, BytesReference> diskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
-                .setKeyType(Key.class)
+            EhCacheDiskCachingTier<Key, BytesReference> diskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>().setKeyType(
+                Key.class
+            )
                 .setValueType(BytesReference.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setSettings(settings)
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
index 139e82b274d34..27b5c6db74549 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
@@ -10,8 +10,11 @@
 
 import org.opensearch.common.Randomness;
 import org.opensearch.common.cache.RemovalListener;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.env.NodeEnvironment;
@@ -21,7 +24,9 @@
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.UUID;
@@ -42,6 +47,7 @@ public void testBasicGetAndPut() throws IOException {
                 .setValueType(String.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setSettings(settings)
+                .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
@@ -77,6 +83,7 @@ public void testBasicGetAndPutBytesReference() throws Exception {
                 .setValueType(BytesReference.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setSettings(settings)
+                .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES * 2) // bigger so no evictions happen
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
@@ -112,6 +119,7 @@ public void testConcurrentPut() throws Exception {
                 .setValueType(String.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setSettings(settings)
+                .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
@@ -155,6 +163,7 @@ public void testEhcacheParallelGets() throws Exception {
                 .setValueType(String.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setSettings(settings)
+                .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
@@ -200,6 +209,7 @@ public void testEhcacheKeyIterator() throws Exception {
                 .setValueType(String.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setSettings(settings)
+                .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setSettingPrefix(SETTING_PREFIX)
@@ -236,6 +246,7 @@ public void testCompute() throws Exception {
                 .setValueType(String.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setSettings(settings)
+                .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setSettingPrefix(SETTING_PREFIX)
@@ -275,40 +286,13 @@ public boolean isLoaded() {
         }
     }
 
-    /*public void testThresholdPolicy() throws Exception {
-        long slowTookTimeNanos = 10000000000L; // 10 seconds
-        BytesReference slowResult = DiskTierTookTimePolicyTests.getQSRBytesReference(slowTookTimeNanos);
-
-        long fastTookTimeNanos = 100000L; // 100 microseconds
-        BytesReference fastResult = DiskTierTookTimePolicyTests.getQSRBytesReference(fastTookTimeNanos);
-
-        long thresholdMillis = 10;
-
-        // For this unit test, set the policy's threshold directly rather than from cluster settings
-        ClusterSettings dummyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        DiskTierTookTimePolicy policy = new DiskTierTookTimePolicy(Settings.EMPTY, dummyClusterSettings);
-        policy.setThreshold(new TimeValue(thresholdMillis));
-
-        Settings settings = Settings.builder().build();
-        try (NodeEnvironment env = newNodeEnvironment(settings)) {
-            EhCacheDiskCachingTier<String, BytesReference> tier = new EhCacheDiskCachingTier.Builder<String, BytesReference>()
-                .setKeyType(String.class)
-                .setValueType(BytesReference.class)
-                .setExpireAfterAccess(TimeValue.MAX_VALUE)
-                .setSettings(settings)
-                .setThreadPoolAlias("ehcacheTest")
-                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
-                .setSettingPrefix(SETTING_PREFIX)
-                .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
-                .withPolicy(policy)
-                .build();
-            tier.put("slow", slowResult);
-            assertEquals(slowResult, tier.get("slow")); // key "slow" is found because the policy accepted it
-            tier.put("fast", fastResult);
-            assertNull(tier.get("fast")); // key "fast" -> null because the policy rejected it
-            tier.close();
-        }
-    }*/
+    private ClusterSettings getClusterSettings() {
+        HashSet<Setting<?>> clusterSettingsSet = new HashSet<>();
+        for (Setting<?> s : ClusterSettings.FEATURE_FLAGGED_CLUSTER_SETTINGS.get(List.of(FeatureFlags.TIERED_CACHING))) {
+            clusterSettingsSet.add(s);
+        } // :(
+        return new ClusterSettings(Settings.EMPTY, clusterSettingsSet);
+    }
 
     private RemovalListener<String, String> removalListener(AtomicInteger counter) {
         return notification -> counter.incrementAndGet();
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index 31aec942b2c9d..f1c1caf11676b 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -68,7 +68,11 @@ public void testInit() {
 
     public void testTransformationLogic() throws Exception {
         int modulo = (int) Math.pow(2, 29);
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L, DUMMY_CLUSTER_SETTINGS);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(
+            RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE,
+            0L,
+            DUMMY_CLUSTER_SETTINGS
+        );
         int offset = 3;
         for (int i = 0; i < 4; i++) { // after this we run into max value, but thats not a flaw with the class design
             int posValue = i * modulo + offset;
@@ -84,7 +88,11 @@ public void testTransformationLogic() throws Exception {
             assertTrue(kls.getInternalRepresentation(value) < modulo);
             assertTrue(kls.getInternalRepresentation(value) >= 0);
         }
-        RBMIntKeyLookupStore no_modulo_kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L, DUMMY_CLUSTER_SETTINGS);
+        RBMIntKeyLookupStore no_modulo_kls = new RBMIntKeyLookupStore(
+            RBMIntKeyLookupStore.KeystoreModuloValue.NONE,
+            0L,
+            DUMMY_CLUSTER_SETTINGS
+        );
         Random rand = Randomness.get();
         for (int i = 0; i < 100; i++) {
             int val = rand.nextInt();
@@ -93,9 +101,17 @@ public void testTransformationLogic() throws Exception {
     }
 
     public void testContains() throws Exception {
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L, DUMMY_CLUSTER_SETTINGS);
-        RBMIntKeyLookupStore noModuloKls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L, DUMMY_CLUSTER_SETTINGS);
-        for (int i = 0; i < kls.REFRESH_SIZE_EST_INTERVAL + 1000; i++) {
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(
+            RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE,
+            0L,
+            DUMMY_CLUSTER_SETTINGS
+        );
+        RBMIntKeyLookupStore noModuloKls = new RBMIntKeyLookupStore(
+            RBMIntKeyLookupStore.KeystoreModuloValue.NONE,
+            0L,
+            DUMMY_CLUSTER_SETTINGS
+        );
+        for (int i = 0; i < RBMIntKeyLookupStore.REFRESH_SIZE_EST_INTERVAL + 1000; i++) {
             // set upper bound > number of elements to trigger a size check, ensuring we test that too
             kls.add(i);
             assertTrue(kls.contains(i));
@@ -123,7 +139,11 @@ public void testAddingStatsGetters() throws Exception {
     public void testRegenerateStore() throws Exception {
         int numToAdd = 10000000;
         Random rand = Randomness.get();
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L, DUMMY_CLUSTER_SETTINGS);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(
+            RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE,
+            0L,
+            DUMMY_CLUSTER_SETTINGS
+        );
         for (int i = 0; i < numToAdd; i++) {
             kls.add(i);
         }
@@ -189,7 +209,11 @@ public void testMemoryCapBlocksAdd() throws Exception {
     public void testConcurrency() throws Exception {
         Random rand = Randomness.get();
         for (int j = 0; j < 5; j++) { // test with different numbers of threads
-            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L, DUMMY_CLUSTER_SETTINGS);
+            RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(
+                RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE,
+                0L,
+                DUMMY_CLUSTER_SETTINGS
+            );
             int numThreads = rand.nextInt(50) + 1;
             ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(numThreads);
             // In this test we want to add the first 200K numbers and check they're all correctly there.
@@ -267,7 +291,11 @@ public void testRemoveNoCollisions() throws Exception {
     public void testRemoveWithCollisions() throws Exception {
         int modulo = (int) Math.pow(2, 26);
         long memCap = 100L * BYTES_IN_MB;
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX, memCap, DUMMY_CLUSTER_SETTINGS);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(
+            RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX,
+            memCap,
+            DUMMY_CLUSTER_SETTINGS
+        );
         for (int i = 0; i < 10; i++) {
             kls.add(i);
             if (i % 2 == 1) {
@@ -310,7 +338,11 @@ public void testRemoveWithCollisions() throws Exception {
     }
 
     public void testNullInputs() throws Exception {
-        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L, DUMMY_CLUSTER_SETTINGS);
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(
+            RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE,
+            0L,
+            DUMMY_CLUSTER_SETTINGS
+        );
         assertFalse(kls.add(null));
         assertFalse(kls.contains(null));
         assertEquals(0, (int) kls.getInternalRepresentation(null));
@@ -432,10 +464,10 @@ public void testSetMemSizeCap() throws Exception {
         }
         long memSize = kls.getMemorySizeInBytes();
         assertEquals(0, kls.getMemorySizeCapInBytes());
-        kls.setMemSizeCap(new ByteSizeValue(memSize/2, ByteSizeUnit.BYTES));
+        kls.setMemSizeCap(new ByteSizeValue(memSize / 2, ByteSizeUnit.BYTES));
         // check the keystore is now full and has its lower cap
         assertTrue(kls.isFull());
-        assertEquals(memSize/2, kls.getMemorySizeCapInBytes());
+        assertEquals(memSize / 2, kls.getMemorySizeCapInBytes());
         assertFalse(kls.add(rand.nextInt()));
     }
 }

From 216d99d0d656cd1e5553fc4b28007577ec29f1f6 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 11 Dec 2023 09:30:39 -0800
Subject: [PATCH 104/120] changed dummy disk cache size from 1 MB -> 10 GB

---
 distribution/src/config/opensearch.yml               | 12 ++++++------
 .../org/opensearch/indices/IndicesRequestCache.java  |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml
index b0f801fccb155..72c81a5d956ee 100644
--- a/distribution/src/config/opensearch.yml
+++ b/distribution/src/config/opensearch.yml
@@ -133,10 +133,10 @@ ${path.logs}
 # opensearch.experimental.feature.tiered_caching.enabled: false
 #
 # Settings related to tiered caching:
-# index.requests.cache.tiered.disk.keystore.size: 0.05%
-# index.requests.cache.tiered.disk.stale_cleanup_threshold: 0.5
-# indices.requests.cache.tiered.disk.ehcache.min_threads: 2
-# indices.requests.cache.tiered.disk.ehcache.max_threads: 2
-# indices.requests.cache.tiered.disk.ehcache.write_concurrency: 2
-# indices.requests.cache.tiered.disk.ehcache.segments: 16
+#index.requests.cache.tiered.disk.keystore.size: 0.05%
+#index.requests.cache.tiered.disk.stale_cleanup_threshold: 0.5
+#indices.requests.cache.tiered.disk.ehcache.min_threads: 2
+#indices.requests.cache.tiered.disk.ehcache.max_threads: 2
+#indices.requests.cache.tiered.disk.ehcache.write_concurrency: 2
+#indices.requests.cache.tiered.disk.ehcache.segments: 16
 #
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 750f114ccee15..2f0cc724abcce 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -148,7 +148,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
 
         if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
             // enabling this for testing purposes. Remove/tweak!!
-            long CACHE_SIZE_IN_BYTES = 1000000L;
+            long CACHE_SIZE_IN_BYTES = 10000000000L; // 10 GB - change when integrating Sagar's code
             String SETTING_PREFIX = "indices.request.cache";
             String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
 

From 8db79e07b467d278b601f3feea6da412f648fc7d Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 11 Dec 2023 16:05:07 -0800
Subject: [PATCH 105/120] added first draft of dynamic disk tier enable/disable
 logic

---
 .../common/cache/tier/DiskTierProvider.java   | 16 +++++
 .../common/cache/tier/TieredCacheService.java |  2 -
 .../TieredCacheSpilloverStrategyService.java  | 71 +++++++++++++++++--
 .../common/settings/ClusterSettings.java      |  2 +
 .../indices/IndicesRequestCache.java          | 62 +++++++++-------
 5 files changed, 117 insertions(+), 36 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java
new file mode 100644
index 0000000000000..a98a728200bef
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java
@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * An interface that provides new instances of a DiskTier.
+ */
+public interface DiskTierProvider<K, V> {
+    DiskCachingTier<K, V> getDiskTier();
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
index 83c76656786bc..31d58510206f0 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheService.java
@@ -31,6 +31,4 @@ public interface TieredCacheService<K, V> {
     OnHeapCachingTier<K, V> getOnHeapCachingTier();
 
     Optional<DiskCachingTier<K, V>> getDiskCachingTier();
-    void addDiskTier(DiskCachingTier<K, V> newTier);
-    void removeDiskTier();
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index e18c15f7ce063..6ab49e51987a2 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -13,6 +13,7 @@
 import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.indices.IndicesRequestCache;
 import org.opensearch.indices.IndicesService;
 
@@ -44,14 +45,22 @@ public class TieredCacheSpilloverStrategyService<K, V> implements TieredCacheSer
      */
     private List<CachingTier<K, V>> cachingTierList;
     private final List<CacheTierPolicy<V>> policies;
+    private boolean diskTierInUse; // True when we are using the disk tier, false when it is deactivated but not deleted
+    private DiskTierProvider<K, V> diskTierProvider; // Used to obtain new instances of DiskTier when the setting is dynamically turned on
 
     private TieredCacheSpilloverStrategyService(Builder<K, V> builder) {
         this.onHeapCachingTier = Objects.requireNonNull(builder.onHeapCachingTier);
         this.diskCachingTier = Optional.ofNullable(builder.diskCachingTier);
+        diskTierInUse = diskCachingTier.isPresent();
         this.tieredCacheEventListener = Objects.requireNonNull(builder.tieredCacheEventListener);
         this.cachingTierList = this.diskCachingTier.map(diskTier -> Arrays.asList(onHeapCachingTier, diskTier))
             .orElse(List.of(onHeapCachingTier));
         this.policies = Objects.requireNonNull(builder.policies);
+        ClusterSettings clusterSettings = Objects.requireNonNull(builder.clusterSettings);
+        if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
+            clusterSettings.addSettingsUpdateConsumer(IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED, this::toggleDiskTierEnabled);
+            this.diskTierProvider = Objects.requireNonNull(builder.diskTierProvider);
+        }
         setRemovalListeners();
     }
 
@@ -200,20 +209,56 @@ private Function<K, CacheValue<V>> getValueFromTierCache(boolean trackStats) {
     /**
      * Dynamically add a new disk tier.
      */
-    public void addDiskTier(DiskCachingTier<K, V> newTier) {
-        assert getDiskCachingTier().isEmpty();
+    private void addNewDiskTier(DiskCachingTier<K, V> newTier) {
+        assert cachingTierList.size() == 1 && diskCachingTier.isEmpty();
+        // list only contains heap tier, and there is no inactive disk tier
         diskCachingTier = Optional.of(newTier);
         cachingTierList = List.of(onHeapCachingTier, diskCachingTier.get());
+        diskTierInUse = true;
     }
 
     /**
-     * Dynamically remove an existing disk tier.
+     * Dynamically reenable an existing disk tier that has been disabled but not deleted.
      */
-    public void removeDiskTier() {
-        assert getDiskCachingTier().isPresent();
-        diskCachingTier.get().close();
-        diskCachingTier = Optional.empty();
+    private void enableInactiveDiskTier() {
+        assert cachingTierList.size() == 1 && diskCachingTier.isPresent();
+        // list only contains heap tier, and there is an inactive disk tier we can reactivate
+        cachingTierList = List.of(onHeapCachingTier, diskCachingTier.get());
+        diskTierInUse = true;
+    }
+
+    /**
+     * Dynamically disable an existing disk tier.
+     */
+    private void disableDiskTier() {
+        // We want to stop searching in or adding to the disk tier, but we don't delete its contents until cache clear API is run.
+        // This is consistent with the heap tier's behavior.
+        // To do this, we remove the disk tier from cachingTierList but not from diskCachingTier.
+        assert cachingTierList.size() == 2 && diskTierInUse; // Contains both heap tier and disk tier
         cachingTierList = List.of(onHeapCachingTier);
+        diskTierInUse = false; // disk tier is now inactive
+    }
+
+    /**
+     * A function to run when disk tier enabled setting (IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED)
+     * is enabled/disabled dynamically.
+     * @param enableDiskTier the new value of the enable disk tier setting
+     */
+    private void toggleDiskTierEnabled(boolean enableDiskTier) {
+        if (enableDiskTier && !diskTierInUse) {
+            // Not currently using a disk tier.
+            if (diskCachingTier.isPresent()) {
+                // There is a deactivated disk tier already instantiated. Reactivate this one rather than creating a new one
+                enableInactiveDiskTier();
+            } else {
+                // There is no disk tier to reactivate, create a new one now.
+                DiskCachingTier<K, V> newTier = diskTierProvider.getDiskTier();
+                addNewDiskTier(newTier);
+            }
+        } else if (!enableDiskTier && diskTierInUse) {
+            // If a disk tier existed before, stop using it. Do not delete it or its contents until cache clear API is run
+            disableDiskTier();
+        }
     }
 
     /**
@@ -226,6 +271,8 @@ public static class Builder<K, V> {
         private DiskCachingTier<K, V> diskCachingTier;
         private TieredCacheEventListener<K, V> tieredCacheEventListener;
         private ArrayList<CacheTierPolicy<V>> policies = new ArrayList<>();
+        private ClusterSettings clusterSettings;
+        private DiskTierProvider<K, V> diskTierProvider;
 
         public Builder() {}
 
@@ -255,6 +302,16 @@ public Builder<K, V> withPolicies(List<CacheTierPolicy<V>> policiesList) {
             return this;
         }
 
+        public Builder<K, V> setClusterSettings(ClusterSettings clusterSettings) {
+            this.clusterSettings = clusterSettings;
+            return this;
+        }
+
+        public Builder<K, V> setDiskTierProvider(DiskTierProvider<K, V> provider) {
+            this.diskTierProvider = provider;
+            return this;
+        }
+
         public TieredCacheSpilloverStrategyService<K, V> build() {
             return new TieredCacheSpilloverStrategyService<K, V>(this);
         }
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 4f09d04557178..c9e9d09b1fa27 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -722,6 +722,8 @@ public void apply(Settings value, Settings current, Settings previous) {
         ),
         List.of(FeatureFlags.TIERED_CACHING),
         List.of(
+            IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED,
+            IndicesRequestCache.INDICES_CACHE_DISK_TIER_SIZE,
             RBMIntKeyLookupStore.INDICES_CACHE_KEYSTORE_SIZE,
             EhCacheDiskCachingTier.REQUEST_CACHE_DISK_STALE_KEY_THRESHOLD,
             EhCacheDiskCachingTier.REQUEST_CACHE_DISK_MIN_THREADS,
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 16e9a8595d9a9..6765d7a01a630 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -43,6 +43,8 @@
 import org.opensearch.common.cache.tier.BytesReferenceSerializer;
 import org.opensearch.common.cache.tier.CachePolicyInfoWrapper;
 import org.opensearch.common.cache.tier.CacheValue;
+import org.opensearch.common.cache.tier.DiskTierProvider;
+import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
 import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
 import org.opensearch.common.cache.tier.OnHeapCachingTier;
 import org.opensearch.common.cache.tier.OpenSearchOnHeapCache;
@@ -94,14 +96,14 @@
  *
  * @opensearch.internal
  */
-public final class IndicesRequestCache implements TieredCacheEventListener<IndicesRequestCache.Key, BytesReference>, Closeable {
+public final class IndicesRequestCache implements TieredCacheEventListener<IndicesRequestCache.Key, BytesReference>, Closeable, DiskTierProvider<IndicesRequestCache.Key, BytesReference> {
 
     private static final Logger logger = LogManager.getLogger(IndicesRequestCache.class);
 
     /**
      * A setting to enable or disable request caching on an index level. Its dynamic by default
      * since we are checking on the cluster state IndexMetadata always.
-     * Note that FeatureFlags.TIERED_CACHING must also be on to enable tiered caching.
+     *
      */
     public static final Setting<Boolean> INDEX_CACHE_REQUEST_ENABLED_SETTING = Setting.boolSetting(
         "index.requests.cache.enable",
@@ -120,6 +122,11 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         Property.NodeScope
     );
 
+    /**
+     * A setting to dynamically enable/disable the disk tier. Note that FeatureFlags.TIERED_CACHING, which is static,
+     * must also be true to enable this and all the below settings.
+     */
+
     public static final Setting<Boolean> INDICES_CACHE_DISK_TIER_ENABLED = Setting.boolSetting(
         "index.requests.cache.tiered.disk.enable",
         false,
@@ -129,7 +136,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
 
     public static final Setting<ByteSizeValue> INDICES_CACHE_DISK_TIER_SIZE = Setting.memorySizeSetting(
         "index.requests.cache.tiered.disk.size",
-        new ByteSizeValue(1, ByteSizeUnit.GB), // TODO: This is a random value - what should it be??
+        new ByteSizeValue(10, ByteSizeUnit.GB), // TODO: This is a random value - what should it be??
         Property.NodeScope
     );
 
@@ -154,11 +161,15 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
     private final TieredCacheService<Key, BytesReference> tieredCacheService;
     private final IndicesService indicesService;
     private final Settings settings;
+    private final ClusterSettings clusterSettings;
 
     IndicesRequestCache(Settings settings, IndicesService indicesService, ClusterSettings clusterSettings) {
         this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
         this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
         long sizeInBytes = size.getBytes();
+        this.indicesService = indicesService;
+        this.settings = settings;
+        this.clusterSettings = clusterSettings;
 
         // Initialize onHeap cache tier first.
         OnHeapCachingTier<Key, BytesReference> openSearchOnHeapCache = new OpenSearchOnHeapCache.Builder<Key, BytesReference>().setWeigher(
@@ -173,9 +184,22 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
             }
         };
 
-        if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING) && ) {
-            EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = getDefaultDiskTier(settings);
+        // Initialize tiered cache service.
+        TieredCacheSpilloverStrategyService.Builder<Key, BytesReference> tieredCacheServiceBuilder =
+            new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>().setOnHeapCachingTier(openSearchOnHeapCache)
+                .setTieredCacheEventListener(this);
+
+        if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
+            if (clusterSettings.get(INDICES_CACHE_DISK_TIER_ENABLED)) {
+                EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = getDiskTier();
+                tieredCacheServiceBuilder.setOnDiskCachingTier(ehcacheDiskTier);
+            }
+            // if the disk tier dynamic setting is disabled, still provide the necessary things to turn it on
+            tieredCacheServiceBuilder.setDiskTierProvider(this);
+            tieredCacheServiceBuilder.withPolicy(new DiskTierTookTimePolicy(settings, clusterSettings, transformationFunction));
         }
+        tieredCacheService = tieredCacheServiceBuilder.build();
+
     }
 
     @Override
@@ -473,19 +497,22 @@ int numRegisteredCloseListeners() { // for testing
 
     /**
      * Creates a new disk tier instance. Should only be run if the instance will actually be used!
-     * @param settings The settings to use
+     * Otherwise, it may not be closed properly.
      * @return A new disk tier instance
      */
-    public EhCacheDiskCachingTier<Key, BytesReference> getDefaultDiskTier(Settings settings) {
+    @Override
+    public EhCacheDiskCachingTier<Key, BytesReference> getDiskTier() {
+        assert FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING);
         String SETTING_PREFIX = "indices.request.cache";
         long CACHE_SIZE_IN_BYTES = INDICES_CACHE_DISK_TIER_SIZE.get(settings).getBytes();
         String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
 
-        EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
+        return new EhCacheDiskCachingTier.Builder<Key, BytesReference>()
             .setKeyType(Key.class)
             .setValueType(BytesReference.class)
             .setExpireAfterAccess(TimeValue.MAX_VALUE) // TODO: Is this meant to be the same as IRC expire or different?
             .setSettings(settings)
+            .setClusterSettings(clusterSettings)
             .setThreadPoolAlias("ehcacheThreadpool")
             .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
             .setStoragePath(STORAGE_PATH)
@@ -493,24 +520,5 @@ public EhCacheDiskCachingTier<Key, BytesReference> getDefaultDiskTier(Settings s
             .setKeySerializer(new IRCKeyWriteableSerializer(this))
             .setValueSerializer(new BytesReferenceSerializer())
             .build();
-        return ehcacheDiskTier;
-    }
-
-    /**
-     * A function to run when disk tier enabled setting (IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED)
-     * is enabled/disabled dynamically.
-     * This function can't go in the TieredCacheSpilloverStrategyService because of issues with generics.
-     * @param enableDiskTier the new value of the enable disk tier setting
-     */
-    private void toggleDiskTierEnabled(boolean enableDiskTier) {
-        if (enableDiskTier && tieredCacheService.getDiskCachingTier().isEmpty()) {
-            // If no disk tier existed before, create one now. Otherwise do nothing
-            EhCacheDiskCachingTier<Key, BytesReference> diskCachingTier = getDefaultDiskTier(settings);
-            tieredCacheService.addDiskTier(diskCachingTier);
-
-        } else if (!enableDiskTier && tieredCacheService.getDiskCachingTier().isPresent()){
-            // If a disk tier existed before, delete it (losing all the cached values). Otherwise do nothing
-            tieredCacheService.removeDiskTier();
-        }
     }
 }

From 153c57510ca191a526590c5139f045f02aca170c Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 12 Dec 2023 12:50:05 -0800
Subject: [PATCH 106/120] added IT testing toggling disk tier dynamic setting

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../IndicesRequestCacheDiskTierIT.java        | 162 +++++++++++++++---
 .../indices/IndicesRequestCacheIT.java        |  11 ++
 .../TieredCacheSpilloverStrategyService.java  |   2 +-
 .../indices/IndicesRequestCache.java          |   6 +-
 4 files changed, 157 insertions(+), 24 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index f14f315b236a5..5a47717cec18d 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.indices;
 
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexMetadata;
@@ -68,32 +69,16 @@ public void testDiskTierStats() throws Exception {
         String node = internalCluster().startNode(
             Settings.builder()
                 .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
-                .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of
-                                                                                                      // took time
+                .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+                .put(IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED.getKey(), true)
         );
         Client client = client(node);
-
-        Settings.Builder indicesSettingBuilder = Settings.builder()
-            .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
-
-        assertAcked(
-            client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(indicesSettingBuilder).get()
-        );
-        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
-        ensureSearchable("index");
-        SearchResponse resp;
-
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
-        int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
-        assertTrue(heapSizeBytes > requestSize);
-        // If this fails, increase heapSizeBytes! We can't adjust it after getting the size of one query
-        // as the cache size setting is not dynamic
+        startIndex(client);
+        int requestSize = getRequestSize(client, heapSizeBytes);
 
         int numOnDisk = 5;
         int numRequests = heapSizeBytes / requestSize + numOnDisk;
-        System.out.println("Size " + requestSize + " numRequests " + numRequests);
+        SearchResponse resp;
         for (int i = 1; i < numRequests; i++) {
             resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
             assertSearchResponse(resp);
@@ -123,6 +108,141 @@ public void testDiskTierStats() throws Exception {
         assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, tookTimeSoFar);
     }
 
+    public void testTogglingDiskTierSetting() throws Exception {
+        // For this test, feature flag tiered caching setting is on, but we toggle IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED
+        // and cover all possible transitions
+        // start with it off (default), then turn it on dynamically
+
+        // setup
+        int heapSizeBytes = 5000;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+                .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+        );
+        Client client = client(node);
+        startIndex(client);
+        int requestSize = getRequestSize(client, heapSizeBytes);
+
+        // Make more queries than fit in the heap cache, and confirm these do not spill over to a disk tier, as we haven't made one yet.
+
+        int numEvicted = 5;
+        int numRequests = heapSizeBytes / requestSize + numEvicted;
+        SearchResponse resp;
+        for (int i = 1; i < numRequests; i++) {
+            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+            assertSearchResponse(resp);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, 0, TierType.DISK, false);
+        }
+        IndicesRequestCacheIT.assertNumEvictions(client, "index", numEvicted, TierType.ON_HEAP);
+        // The heap tier should now contain requests # numEvicted through numRequests. 0 through numEvicted have been evicted and lost.
+
+        // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = false. Nothing should happen
+        setDiskCacheEnabled(client, false);
+        // insert check for diskTierEnabled here once I add that to API
+
+        // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = true, creating a new disk tier
+        setDiskCacheEnabled(client, true);
+
+        // Make some new requests. These should cause evictions from the heap tier which spill over to the disk tier (requests between numEvicted and numEvicted + numOnDisk)
+        int numOnDisk = 3;
+        for (int i = numRequests; i < numRequests + numOnDisk; i++) {
+            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+            assertSearchResponse(resp);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i - numRequests + 1, TierType.DISK, false);
+        }
+        IndicesRequestCacheIT.assertNumEvictions(client, "index", numEvicted + numOnDisk, TierType.ON_HEAP);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", heapSizeBytes / requestSize, TierType.ON_HEAP);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
+        // Confirm request # numEvicted + 1 is a disk tier hit
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + (numEvicted + 1))).get();
+        assertSearchResponse(resp);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + 1, TierType.ON_HEAP, false);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 1, numOnDisk, TierType.DISK, false);
+        // assert disk tier in use - future
+
+        // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = true. Nothing should happen
+        setDiskCacheEnabled(client, true);
+        // insert check here in future
+
+        // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = false. The disk tier should be deactivated but not deleted
+        setDiskCacheEnabled(client, false);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
+        // assert disk tier not in use - future
+
+        // Make some new requests. These should cause evictions from the heap tier (numEvicted + numOnDisk through numEvicted + numOnDisk + numNewRequests), but not reach the deactivated disk tier
+        int numNewRequests = 5;
+        for (int i = numRequests + numOnDisk; i < numRequests + numOnDisk + numNewRequests; i++) {
+            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+            assertSearchResponse(resp);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 2, TierType.ON_HEAP, false);
+            IndicesRequestCacheIT.assertCacheState(client, "index", 1, numOnDisk, TierType.DISK, false);
+        }
+        IndicesRequestCacheIT.assertNumEvictions(client, "index", numEvicted + numOnDisk + numNewRequests, TierType.ON_HEAP);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", heapSizeBytes / requestSize, TierType.ON_HEAP);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
+        // Now the heap tier contains requests numEvicted + numOnDisk + numNewRequests through numEvicted + numOnDisk + numNewRequests + num
+
+        // Check a request (# numEvicted) which was on the disk tier is now a cache miss. This evicts #
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + numEvicted)).get();
+        assertSearchResponse(resp);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + numNewRequests + 2, TierType.ON_HEAP, false);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 1, numOnDisk, TierType.DISK, false);
+
+
+        // Re-enable the disk tier. We should get the same previously deactivated disk tier, not a new one
+        setDiskCacheEnabled(client, true);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
+        // Request # numEvicted + 2 should be a disk tier hit
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + (numEvicted + 2))).get();
+        assertSearchResponse(resp);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + numNewRequests + 3, TierType.ON_HEAP, false);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 2, numOnDisk, TierType.DISK, false);
+        // New queries should cause spillover to the disk tier again
+        int reqNum = numRequests + numOnDisk + numNewRequests;
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + reqNum)).get();
+        assertSearchResponse(resp);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + numNewRequests + 4, TierType.ON_HEAP, false);
+        IndicesRequestCacheIT.assertCacheState(client, "index", 2, numOnDisk + 1, TierType.DISK, false);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk + 1, TierType.DISK);
+
+        // TODO: Once Kiran's cache clear PR is done, add a case where we deactivate, clear disk cache, reactivate, and see a new empty disk tier.
+    }
+
+    private void startIndex(Client client) throws Exception {
+        Settings.Builder indicesSettingBuilder = Settings.builder()
+            .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+        assertAcked(
+            client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(indicesSettingBuilder).get()
+        );
+        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+        ensureSearchable("index");
+    }
+
+    private int getRequestSize(Client client, int heapSizeBytes) throws Exception {
+        // Setup function to get the maximum number of entries of the form "hello" + i
+        // that will fit in the heap cache, by making a query and checking size afterwards
+
+        SearchResponse resp;
+        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
+        int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
+        assertTrue(heapSizeBytes > requestSize);
+        // If this fails, increase heapSizeBytes! We can't adjust it after getting the size of one query
+        // as the cache size setting is not dynamic
+        return requestSize;
+    }
+
+    private void setDiskCacheEnabled(Client client, boolean newSetting) {
+        ClusterUpdateSettingsRequest clusterSettingUpdate = new ClusterUpdateSettingsRequest();
+        clusterSettingUpdate.persistentSettings(Settings.builder().put(IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED.getKey(), newSetting));
+        assertAcked(client.admin().cluster().updateSettings(clusterSettingUpdate).actionGet());
+    }
+
     private long getCacheSizeBytes(Client client, String index, TierType tierType) {
         RequestCacheStats requestCacheStats = client.admin()
             .indices()
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index f07f667f71d5b..ff098e488024e 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -732,4 +732,15 @@ protected static void assertNumCacheEntries(Client client, String index, long ex
         assertEquals(expectedEntries, requestCacheStats.getEntries(tierType));
     }
 
+    protected static void assertNumEvictions(Client client, String index, long expectedEvictions, TierType tierType) {
+        RequestCacheStats requestCacheStats = client.admin()
+            .indices()
+            .prepareStats(index)
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+        assertEquals(expectedEvictions, requestCacheStats.getEvictions(tierType));
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 6ab49e51987a2..2b7b06347c61b 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -146,7 +146,7 @@ public void onRemoval(RemovalNotification<K, V> notification) {
         if (RemovalReason.EVICTED.equals(notification.getRemovalReason())) {
             switch (notification.getTierType()) {
                 case ON_HEAP:
-                    if (checkPolicies(notification.getValue())) {
+                    if (checkPolicies(notification.getValue()) && diskTierInUse) {
                         diskCachingTier.ifPresent(diskTier -> {
                             diskTier.put(notification.getKey(), notification.getValue());
                             tieredCacheEventListener.onCached(notification.getKey(), notification.getValue(), TierType.DISK);
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 6765d7a01a630..67ce6a4471a41 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -186,8 +186,10 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
 
         // Initialize tiered cache service.
         TieredCacheSpilloverStrategyService.Builder<Key, BytesReference> tieredCacheServiceBuilder =
-            new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>().setOnHeapCachingTier(openSearchOnHeapCache)
-                .setTieredCacheEventListener(this);
+            new TieredCacheSpilloverStrategyService.Builder<Key, BytesReference>()
+                .setOnHeapCachingTier(openSearchOnHeapCache)
+                .setTieredCacheEventListener(this)
+                .setClusterSettings(clusterSettings);
 
         if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
             if (clusterSettings.get(INDICES_CACHE_DISK_TIER_ENABLED)) {

From 52dad7a26554303b121f11752408d7b9ca52e6a1 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 12 Dec 2023 14:04:22 -0800
Subject: [PATCH 107/120] cleanup

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../opensearch/indices/IndicesRequestCacheDiskTierIT.java    | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index 5a47717cec18d..d0849cd28343d 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -140,7 +140,6 @@ public void testTogglingDiskTierSetting() throws Exception {
 
         // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = false. Nothing should happen
         setDiskCacheEnabled(client, false);
-        // insert check for diskTierEnabled here once I add that to API
 
         // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = true, creating a new disk tier
         setDiskCacheEnabled(client, true);
@@ -161,19 +160,17 @@ public void testTogglingDiskTierSetting() throws Exception {
         assertSearchResponse(resp);
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + 1, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 1, numOnDisk, TierType.DISK, false);
-        // assert disk tier in use - future
 
         // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = true. Nothing should happen
         setDiskCacheEnabled(client, true);
-        // insert check here in future
 
         // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = false. The disk tier should be deactivated but not deleted
         setDiskCacheEnabled(client, false);
         IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
-        // assert disk tier not in use - future
 
         // Make some new requests. These should cause evictions from the heap tier (numEvicted + numOnDisk through numEvicted + numOnDisk + numNewRequests), but not reach the deactivated disk tier
         int numNewRequests = 5;
+        assertTrue(numNewRequests > 3); // necessary for checks later in the test
         for (int i = numRequests + numOnDisk; i < numRequests + numOnDisk + numNewRequests; i++) {
             resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
             assertSearchResponse(resp);

From 1671e2fc3ac82e5b4e08c044a8118f799a17a8d2 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 12 Dec 2023 14:11:09 -0800
Subject: [PATCH 108/120] removed disk tier checks from IT where disk tier is
 not turned on

---
 .../java/org/opensearch/indices/IndicesRequestCacheIT.java   | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index ff098e488024e..02664c86d0967 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -668,7 +668,6 @@ public void testCacheWithInvalidation() throws Exception {
         assertSearchResponse(resp);
         // Should expect hit as here as refresh didn't happen
         assertCacheState(client, "index", 1, 1, TierType.ON_HEAP, false);
-        assertCacheState(client, "index", 0, 1, TierType.DISK, false);
         assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP);
 
         // Explicit refresh would invalidate cache
@@ -678,9 +677,6 @@ public void testCacheWithInvalidation() throws Exception {
         assertSearchResponse(resp);
         // Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh)
         assertCacheState(client, "index", 1, 2, TierType.ON_HEAP, false);
-        assertCacheState(client, "index", 0, 2, TierType.DISK, false);
-
-        // assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP); // Evictions won't be 1 until the cache cleaner runs every minute
     }
 
     protected static void assertCacheState(
@@ -742,5 +738,4 @@ protected static void assertNumEvictions(Client client, String index, long expec
             .getRequestCache();
         assertEquals(expectedEvictions, requestCacheStats.getEvictions(tierType));
     }
-
 }

From 24b5f14591c4ff25c8cbaaa52dfec83fd1eeaa32 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 12 Dec 2023 14:24:39 -0800
Subject: [PATCH 109/120] added checks to disk enable/disable fns, condensed IT

---
 .../IndicesRequestCacheDiskTierIT.java        | 60 +++++++++----------
 .../TieredCacheSpilloverStrategyService.java  |  4 +-
 2 files changed, 29 insertions(+), 35 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index d0849cd28343d..7932a23c11020 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -74,14 +74,12 @@ public void testDiskTierStats() throws Exception {
         );
         Client client = client(node);
         startIndex(client);
-        int requestSize = getRequestSize(client, heapSizeBytes);
+        int requestSize = getRequestSize(client, "index", heapSizeBytes);
 
         int numOnDisk = 5;
         int numRequests = heapSizeBytes / requestSize + numOnDisk;
-        SearchResponse resp;
         for (int i = 1; i < numRequests; i++) {
-            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
-            assertSearchResponse(resp);
+            doHelloSearch(client, "index", i);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
         }
@@ -90,19 +88,19 @@ public void testDiskTierStats() throws Exception {
         long tookTimeSoFar = assertDiskTierSpecificStats(client, "index", 0, -1, 0);
 
         // the first request, for "hello0", should have been evicted to the disk tier
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello0")).get();
+        doHelloSearch(client, "index", 0);
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 1, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 1, numRequests, TierType.DISK, false);
         tookTimeSoFar = assertDiskTierSpecificStats(client, "index", 1, 0, -1);
 
         // We make another actual request that should be in the disk tier. Disk specific stats should keep incrementing
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello1")).get();
+        doHelloSearch(client, "index", 1);
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 2, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 2, numRequests, TierType.DISK, false);
         tookTimeSoFar = assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, -1);
 
         // A final request for something in neither tier shouldn't increment disk specific stats
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + numRequests)).get();
+        doHelloSearch(client, "index", numRequests);
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 3, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 2, numRequests + 1, TierType.DISK, false);
         assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, tookTimeSoFar);
@@ -122,7 +120,7 @@ public void testTogglingDiskTierSetting() throws Exception {
         );
         Client client = client(node);
         startIndex(client);
-        int requestSize = getRequestSize(client, heapSizeBytes);
+        int requestSize = getRequestSize(client, "index", heapSizeBytes);
 
         // Make more queries than fit in the heap cache, and confirm these do not spill over to a disk tier, as we haven't made one yet.
 
@@ -130,8 +128,7 @@ public void testTogglingDiskTierSetting() throws Exception {
         int numRequests = heapSizeBytes / requestSize + numEvicted;
         SearchResponse resp;
         for (int i = 1; i < numRequests; i++) {
-            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
-            assertSearchResponse(resp);
+            doHelloSearch(client, "index", i);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, 0, TierType.DISK, false);
         }
@@ -139,16 +136,15 @@ public void testTogglingDiskTierSetting() throws Exception {
         // The heap tier should now contain requests # numEvicted through numRequests. 0 through numEvicted have been evicted and lost.
 
         // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = false. Nothing should happen
-        setDiskCacheEnabled(client, false);
+        setDiskCacheEnabledSetting(client, false);
 
         // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = true, creating a new disk tier
-        setDiskCacheEnabled(client, true);
+        setDiskCacheEnabledSetting(client, true);
 
         // Make some new requests. These should cause evictions from the heap tier which spill over to the disk tier (requests between numEvicted and numEvicted + numOnDisk)
         int numOnDisk = 3;
         for (int i = numRequests; i < numRequests + numOnDisk; i++) {
-            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
-            assertSearchResponse(resp);
+            doHelloSearch(client, "index", i);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i - numRequests + 1, TierType.DISK, false);
         }
@@ -156,24 +152,22 @@ public void testTogglingDiskTierSetting() throws Exception {
         IndicesRequestCacheIT.assertNumCacheEntries(client, "index", heapSizeBytes / requestSize, TierType.ON_HEAP);
         IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
         // Confirm request # numEvicted + 1 is a disk tier hit
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + (numEvicted + 1))).get();
-        assertSearchResponse(resp);
+        doHelloSearch(client, "index", numEvicted + 1);
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + 1, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 1, numOnDisk, TierType.DISK, false);
 
         // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = true. Nothing should happen
-        setDiskCacheEnabled(client, true);
+        setDiskCacheEnabledSetting(client, true);
 
         // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = false. The disk tier should be deactivated but not deleted
-        setDiskCacheEnabled(client, false);
+        setDiskCacheEnabledSetting(client, false);
         IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
 
         // Make some new requests. These should cause evictions from the heap tier (numEvicted + numOnDisk through numEvicted + numOnDisk + numNewRequests), but not reach the deactivated disk tier
         int numNewRequests = 5;
         assertTrue(numNewRequests > 3); // necessary for checks later in the test
         for (int i = numRequests + numOnDisk; i < numRequests + numOnDisk + numNewRequests; i++) {
-            resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
-            assertSearchResponse(resp);
+            doHelloSearch(client, "index", i);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 2, TierType.ON_HEAP, false);
             IndicesRequestCacheIT.assertCacheState(client, "index", 1, numOnDisk, TierType.DISK, false);
         }
@@ -183,24 +177,20 @@ public void testTogglingDiskTierSetting() throws Exception {
         // Now the heap tier contains requests numEvicted + numOnDisk + numNewRequests through numEvicted + numOnDisk + numNewRequests + num
 
         // Check a request (# numEvicted) which was on the disk tier is now a cache miss. This evicts #
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + numEvicted)).get();
-        assertSearchResponse(resp);
+        doHelloSearch(client, "index", numEvicted);
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + numNewRequests + 2, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 1, numOnDisk, TierType.DISK, false);
 
 
         // Re-enable the disk tier. We should get the same previously deactivated disk tier, not a new one
-        setDiskCacheEnabled(client, true);
+        setDiskCacheEnabledSetting(client, true);
         IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
         // Request # numEvicted + 2 should be a disk tier hit
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + (numEvicted + 2))).get();
-        assertSearchResponse(resp);
+        doHelloSearch(client, "index", numEvicted + 2);
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + numNewRequests + 3, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 2, numOnDisk, TierType.DISK, false);
         // New queries should cause spillover to the disk tier again
-        int reqNum = numRequests + numOnDisk + numNewRequests;
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + reqNum)).get();
-        assertSearchResponse(resp);
+        doHelloSearch(client, "index", numRequests + numOnDisk + numNewRequests);
         IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + numNewRequests + 4, TierType.ON_HEAP, false);
         IndicesRequestCacheIT.assertCacheState(client, "index", 2, numOnDisk + 1, TierType.DISK, false);
         IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk + 1, TierType.DISK);
@@ -208,6 +198,12 @@ public void testTogglingDiskTierSetting() throws Exception {
         // TODO: Once Kiran's cache clear PR is done, add a case where we deactivate, clear disk cache, reactivate, and see a new empty disk tier.
     }
 
+    private void doHelloSearch(Client client, String index, int number) {
+        // make a search request of the form "hello" + number
+        SearchResponse resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + number)).get();
+        assertSearchResponse(resp);
+    }
+
     private void startIndex(Client client) throws Exception {
         Settings.Builder indicesSettingBuilder = Settings.builder()
             .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
@@ -221,12 +217,10 @@ private void startIndex(Client client) throws Exception {
         ensureSearchable("index");
     }
 
-    private int getRequestSize(Client client, int heapSizeBytes) throws Exception {
+    private int getRequestSize(Client client, String index, int heapSizeBytes) throws Exception {
         // Setup function to get the maximum number of entries of the form "hello" + i
         // that will fit in the heap cache, by making a query and checking size afterwards
-
-        SearchResponse resp;
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
+        doHelloSearch(client, index, 0);
         int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
         assertTrue(heapSizeBytes > requestSize);
         // If this fails, increase heapSizeBytes! We can't adjust it after getting the size of one query
@@ -234,7 +228,7 @@ private int getRequestSize(Client client, int heapSizeBytes) throws Exception {
         return requestSize;
     }
 
-    private void setDiskCacheEnabled(Client client, boolean newSetting) {
+    private void setDiskCacheEnabledSetting(Client client, boolean newSetting) {
         ClusterUpdateSettingsRequest clusterSettingUpdate = new ClusterUpdateSettingsRequest();
         clusterSettingUpdate.persistentSettings(Settings.builder().put(IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED.getKey(), newSetting));
         assertAcked(client.admin().cluster().updateSettings(clusterSettingUpdate).actionGet());
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 2b7b06347c61b..1f6e50e84d32f 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -210,7 +210,7 @@ private Function<K, CacheValue<V>> getValueFromTierCache(boolean trackStats) {
      * Dynamically add a new disk tier.
      */
     private void addNewDiskTier(DiskCachingTier<K, V> newTier) {
-        assert cachingTierList.size() == 1 && diskCachingTier.isEmpty();
+        assert cachingTierList.size() == 1 && diskCachingTier.isEmpty() && !diskTierInUse;
         // list only contains heap tier, and there is no inactive disk tier
         diskCachingTier = Optional.of(newTier);
         cachingTierList = List.of(onHeapCachingTier, diskCachingTier.get());
@@ -221,7 +221,7 @@ private void addNewDiskTier(DiskCachingTier<K, V> newTier) {
      * Dynamically reenable an existing disk tier that has been disabled but not deleted.
      */
     private void enableInactiveDiskTier() {
-        assert cachingTierList.size() == 1 && diskCachingTier.isPresent();
+        assert cachingTierList.size() == 1 && diskCachingTier.isPresent() && !diskTierInUse;
         // list only contains heap tier, and there is an inactive disk tier we can reactivate
         cachingTierList = List.of(onHeapCachingTier, diskCachingTier.get());
         diskTierInUse = true;

From 1c1a2b5ee6fc2c3ec5d081b897ec158a3f54f506 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 12 Dec 2023 15:11:29 -0800
Subject: [PATCH 110/120] set default disk tier size to 100 mb

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../main/java/org/opensearch/indices/IndicesRequestCache.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 67ce6a4471a41..77c0662445f96 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -136,9 +136,9 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
 
     public static final Setting<ByteSizeValue> INDICES_CACHE_DISK_TIER_SIZE = Setting.memorySizeSetting(
         "index.requests.cache.tiered.disk.size",
-        new ByteSizeValue(10, ByteSizeUnit.GB), // TODO: This is a random value - what should it be??
+        new ByteSizeValue(100, ByteSizeUnit.MB),
         Property.NodeScope
-    );
+    ); // TODO: This is 1% of the minimum EBS size for most EC2 instances. In future set to 1% of actual disk size if possible
 
     public static final Setting<Double> INDICES_CACHE_DISK_STALE_KEY_THRESHOLD = Setting.doubleSetting(
         "index.requests.cache.tiered.disk.stale_cleanup_threshold",

From 2758ee351bb53843664e29221ae14ffd564fabda Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Tue, 12 Dec 2023 16:19:45 -0800
Subject: [PATCH 111/120] invalidateAll() also invalidates keys from inactive
 disk tier

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 .../cache/tier/TieredCacheSpilloverStrategyService.java       | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 1f6e50e84d32f..0d6996daf2e9c 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -121,6 +121,10 @@ public void invalidateAll() {
         for (CachingTier<K, V> cachingTier : cachingTierList) {
             cachingTier.invalidateAll();
         }
+        if (!diskTierInUse && diskCachingTier.isPresent()) {
+            // Invalidate all keys in an inactive disk tier as well
+            diskCachingTier.get().invalidateAll();
+        }
     }
 
     /**

From f8b04259eebbd22c9ee3bd163d10f5dcee089862 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Wed, 13 Dec 2023 15:38:19 -0800
Subject: [PATCH 112/120] set disk threshold policy to 0

---
 .../opensearch/common/cache/tier/DiskTierTookTimePolicy.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
index 5e6f3b2d4be26..f5c52b5f0afd8 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
@@ -30,7 +30,7 @@
 public class DiskTierTookTimePolicy implements CacheTierPolicy<BytesReference> {
     public static final Setting<TimeValue> DISK_TOOKTIME_THRESHOLD_SETTING = Setting.positiveTimeSetting(
         "indices.requests.cache.disk.tooktime.threshold",
-        new TimeValue(10),
+        TimeValue.ZERO,
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
     ); // Set this to TimeValue.ZERO to let all data through

From 8f0b5bcb3f80427166da1f34f04494c3172831ec Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Thu, 14 Dec 2023 17:17:28 -0800
Subject: [PATCH 113/120] refactor cleanCache & cleanDiskCache to share methods

---
 .../indices/IndicesRequestCache.java          | 114 +++++++++---------
 1 file changed, 55 insertions(+), 59 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 600b12a78f3f7..44af6361e95ee 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -526,12 +526,29 @@ public int hashCode() {
      * Logic to clean up in-memory cache.
      */
     synchronized void cleanCache() {
+        cleanCache(TierType.ON_HEAP);
+        tieredCacheService.getOnHeapCachingTier().refresh();
+    }
+
+    /**
+     * Logic to clean up disk based cache.
+     * <p>
+     * TODO: cleanDiskCache is very specific to disk caching tier. We can refactor this to be more generic.
+     */
+    synchronized void cleanDiskCache(double diskCachesCleanThresholdPercent) {
+        if (!canSkipDiskCacheCleanup(diskCachesCleanThresholdPercent)) {
+            cleanCache(TierType.DISK);
+        }
+    }
+
+    private synchronized void cleanCache(TierType cacheType) {
         final Set<CleanupKey> currentKeysToClean = new HashSet<>();
         final Set<Object> currentFullClean = new HashSet<>();
-        /*
-        Stores the keys that need to be removed from keysToClean
-        This is done to avoid ConcurrentModificationException
-        */
+
+            /*
+            Stores the keys that need to be removed from keysToClean
+            This is done to avoid ConcurrentModificationException
+            */
         final Set<CleanupKey> keysCleanedFromAllCaches = new HashSet<>();
 
         for (Map.Entry<CleanupKey, CleanupStatus> entry : keysToClean.entrySet()) {
@@ -543,14 +560,20 @@ synchronized void cleanCache() {
                 continue;
             }
 
-            if (cleanupStatus.cleanedInHeap) continue;
+            if (cacheType == TierType.ON_HEAP && cleanupStatus.cleanedInHeap) continue;
+            if (cacheType == TierType.DISK && cleanupStatus.cleanedOnDisk) continue;
 
             if (needsFullClean(cleanupKey)) {
                 currentFullClean.add(cleanupKey.entity.getCacheIdentity());
             } else {
                 currentKeysToClean.add(cleanupKey);
             }
-            cleanupStatus.cleanedInHeap = true;
+
+            if (cacheType == TierType.ON_HEAP) {
+                cleanupStatus.cleanedInHeap = true;
+            } else if (cacheType == TierType.DISK) {
+                cleanupStatus.cleanedOnDisk = true;
+            }
         }
 
         // Remove keys that have been cleaned from all caches
@@ -561,68 +584,41 @@ synchronized void cleanCache() {
             return;
         }
 
+        CachingTier<Key, BytesReference> cachingTier;
+
+        if (cacheType == TierType.ON_HEAP) {
+            cachingTier = tieredCacheService.getOnHeapCachingTier();
+        } else {
+            cachingTier = tieredCacheService.getDiskCachingTier().get();
+        }
+
         cleanUpKeys(
-            tieredCacheService.getOnHeapCachingTier(),
+            cachingTier,
             currentKeysToClean,
             currentFullClean
         );
-        tieredCacheService.getOnHeapCachingTier().refresh();
     }
 
-    /**
-     * Logic to clean up disk based cache.
-     * <p>
-     * TODO: cleanDiskCache is very specific to disk caching tier. We can refactor this to be more generic.
-     */
-    synchronized void cleanDiskCache(double diskCachesCleanThresholdPercent) {
-        tieredCacheService.getDiskCachingTier().ifPresent(diskCachingTier -> {
-            if (diskCachingTier.count() == 0 || diskCleanupKeysPercentage() < diskCachesCleanThresholdPercent) {
-                if (logger.isDebugEnabled()) {
-                    logger.debug("Skipping disk cache keys cleanup");
-                    return;
-                }
+    private synchronized boolean canSkipDiskCacheCleanup(double diskCachesCleanThresholdPercent) {
+        if (tieredCacheService.getDiskCachingTier().isEmpty()) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Skipping disk cache keys cleanup since disk caching tier is not present");
             }
-            // Categorize keys to be cleaned into currentKeysToClean and currentFullClean
-            final Set<CleanupKey> currentKeysToClean = new HashSet<>();
-            final Set<Object> currentFullClean = new HashSet<>();
-
-            /*
-            Stores the keys that need to be removed from keysToClean
-            This is done to avoid ConcurrentModificationException
-            */
-            final Set<CleanupKey> keysCleanedFromAllCaches = new HashSet<>();
-
-            for (Map.Entry<CleanupKey, CleanupStatus> entry : keysToClean.entrySet()) {
-                CleanupKey cleanupKey = entry.getKey();
-                CleanupStatus cleanupStatus = entry.getValue();
-
-                if (cleanupStatus.cleanedInHeap && cleanupStatus.cleanedOnDisk) {
-                    keysCleanedFromAllCaches.add(cleanupKey);
-                    continue;
-                }
-
-                if (cleanupStatus.cleanedOnDisk) continue;
-
-                if (needsFullClean(cleanupKey)) {
-                    currentFullClean.add(cleanupKey.entity.getCacheIdentity());
-                } else {
-                    currentKeysToClean.add(cleanupKey);
-                }
-                cleanupStatus.cleanedOnDisk = true;
+            return true;
+        }
+        if (tieredCacheService.getDiskCachingTier().get().count() == 0) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Skipping disk cache keys cleanup since disk caching tier is empty");
             }
-
-            // Remove keys that have been cleaned from all caches
-            keysToClean.keySet().removeAll(keysCleanedFromAllCaches);
-
-            // Early exit if no cleanup is needed
-            if (currentKeysToClean.isEmpty() && currentFullClean.isEmpty()) {
-                return;
+            return true;
+        }
+        if (diskCleanupKeysPercentage() < diskCachesCleanThresholdPercent) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Skipping disk cache keys cleanup since the percentage of stale keys in disk cache is less than the threshold");
             }
-            cleanUpKeys(
-                tieredCacheService.getDiskCachingTier().get(),
-                currentKeysToClean,
-                currentFullClean);
-        });
+            return true;
+        }
+        return false;
     }
 
     synchronized double diskCleanupKeysPercentage() {

From 8c996b3a52a26e18072a60114570cde05de56a13 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Thu, 14 Dec 2023 17:17:42 -0800
Subject: [PATCH 114/120] null checks for indexShard

---
 .../java/org/opensearch/indices/IndicesRequestCache.java | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 44af6361e95ee..9a1a1c58c2b97 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -257,6 +257,11 @@ private void updateDiskCleanupKeyToCountMap(CleanupKey cleanupKey, TierType tier
             return;
         }
         IndexShard indexShard = (IndexShard)cleanupKey.entity.getCacheIdentity();
+        if(indexShard == null) {
+            logger.warn("IndexShard is null for CleanupKey: {} while cleaning tier : {}",
+                cleanupKey.readerCacheKeyId, tierType.getStringValue());
+            return;
+        }
         ShardId shardId = indexShard.shardId();
 
         diskCleanupKeyToCountMap
@@ -275,6 +280,10 @@ private void updateDiskCleanupKeyToCountMap(CleanupKey cleanupKey, TierType tier
      */
     private void updateStaleKeysInDiskCount(CleanupKey cleanupKey) {
         IndexShard indexShard = (IndexShard) cleanupKey.entity.getCacheIdentity();
+        if(indexShard == null) {
+            logger.warn("IndexShard is null for CleanupKey: {}",  cleanupKey.readerCacheKeyId);
+            return;
+        }
         ShardId shardId = indexShard.shardId();
 
         ConcurrentMap<String, Integer> countMap = diskCleanupKeyToCountMap.get(shardId);

From 4bd1503ce7c18f9066e84d1ee4ec599ffbb5a5d6 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Thu, 14 Dec 2023 17:18:03 -0800
Subject: [PATCH 115/120] some bugs i found

---
 .../java/org/opensearch/indices/IndicesRequestCache.java    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 9a1a1c58c2b97..227751a668c3d 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -637,7 +637,7 @@ synchronized double diskCleanupKeysPercentage() {
         if (totalKeysInDiskCache == 0 || staleKeysInDiskCount.get() == 0) {
             return 0;
         }
-        return ((double) staleKeysInDiskCount.get() / totalKeysInDiskCache) * 100;
+        return ((double) staleKeysInDiskCount.get() / totalKeysInDiskCache);
     }
 
     synchronized void cleanUpKeys(
@@ -649,7 +649,9 @@ synchronized void cleanUpKeys(
             CleanupKey cleanupKey = new CleanupKey(key.entity, key.readerCacheKeyId);
             if (currentFullClean.contains(key.entity.getCacheIdentity()) || currentKeysToClean.contains(cleanupKey)) {
                 cachingTier.invalidate(key);
-                staleKeysInDiskCount.decrementAndGet();
+                if(cachingTier.getTierType() == TierType.DISK) {
+                    staleKeysInDiskCount.decrementAndGet();
+                }
             }
         }
     }

From 1911fa65f630b54ec4ddfbf89271fdb97ac76481 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Thu, 14 Dec 2023 17:18:55 -0800
Subject: [PATCH 116/120] use assertNumCacheEntries

---
 .../IndicesRequestCacheDiskTierIT.java        | 97 ++-----------------
 1 file changed, 6 insertions(+), 91 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index bbd4ab6e9975d..f85b06ceeac44 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -147,52 +147,18 @@ public void testDiskTierInvalidationByCleanCacheAPI() throws Exception {
         // as the cache size setting is not dynamic
         int numOnDisk = 2;
         int numRequests = heapSizeBytes / requestSize + numOnDisk;
-        RequestCacheStats requestCacheStats = client.admin()
-            .indices()
-            .prepareStats("index")
-            .setRequestCache(true)
-            .get()
-            .getTotal()
-            .getRequestCache();
         for (int i = 1; i < numRequests; i++) {
-            requestCacheStats = client.admin()
-                .indices()
-                .prepareStats("index")
-                .setRequestCache(true)
-                .get()
-                .getTotal()
-                .getRequestCache();
             resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
             assertSearchResponse(resp);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
         }
-
-        requestCacheStats = client.admin()
-            .indices()
-            .prepareStats("index")
-            .setRequestCache(true)
-            .get()
-            .getTotal()
-            .getRequestCache();
-
-        long entries = requestCacheStats.getEntries(TierType.DISK);
-        // make sure we have 2 entries in disk.
-        assertEquals(2, entries);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 2, TierType.DISK);
 
         // call clear cache api
         client.admin().indices().prepareClearCache().setIndices("index").setRequestCache(true).get();
         // fetch the stats again
-        requestCacheStats = client.admin()
-            .indices()
-            .prepareStats("index")
-            .setRequestCache(true)
-            .get()
-            .getTotal()
-            .getRequestCache();
-        entries = requestCacheStats.getEntries(TierType.DISK);
-        // make sure we have 0 entries in disk.
-        assertEquals(0, entries);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 0, TierType.DISK);
     }
 
     // When entire disk tier is stale, test whether cache cleaner cleans up everything from disk
@@ -235,17 +201,7 @@ public void testDiskTierInvalidationByCacheCleanerEntireDiskTier() throws Except
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
         }
 
-        RequestCacheStats requestCacheStats = client.admin()
-            .indices()
-            .prepareStats("index")
-            .setRequestCache(true)
-            .get()
-            .getTotal()
-            .getRequestCache();
-
-        // make sure we have 2 entries in disk.
-         long entries = requestCacheStats.getEntries(TierType.DISK);
-        assertEquals(2, entries);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 2, TierType.DISK);
 
         // index a doc and force refresh so that the cache cleaner can clean the cache
         indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
@@ -260,17 +216,7 @@ public void testDiskTierInvalidationByCacheCleanerEntireDiskTier() throws Except
 
         // by now cache cleaner would have run and cleaned up stale keys
         // fetch the stats again
-        requestCacheStats = client.admin()
-            .indices()
-            .prepareStats("index")
-            .setRequestCache(true)
-            .get()
-            .getTotal()
-            .getRequestCache();
-
-        // make sure we have 0 entries in disk.
-        entries = requestCacheStats.getEntries(TierType.DISK);
-        assertEquals(0, entries);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 0, TierType.DISK);
     }
 
     // When part of disk tier is stale, test whether cache cleaner cleans up only stale items from disk
@@ -313,17 +259,7 @@ public void testDiskTierInvalidationByCacheCleanerPartOfDiskTier() throws Except
             IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
         }
 
-        RequestCacheStats requestCacheStats = client.admin()
-            .indices()
-            .prepareStats("index")
-            .setRequestCache(true)
-            .get()
-            .getTotal()
-            .getRequestCache();
-
-        // make sure we have 2 entries in disk.
-        long entries = requestCacheStats.getEntries(TierType.DISK);
-        assertEquals(2, entries);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 2, TierType.DISK);
 
         // force refresh so that it creates stale keys in the cache for the cache cleaner to pick up.
         flushAndRefresh("index");
@@ -334,17 +270,6 @@ public void testDiskTierInvalidationByCacheCleanerPartOfDiskTier() throws Except
             client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
         }
 
-        // fetch the stats again
-        requestCacheStats = client.admin()
-            .indices()
-            .prepareStats("index")
-            .setRequestCache(true)
-            .get()
-            .getTotal()
-            .getRequestCache();
-
-        entries = requestCacheStats.getEntries(TierType.DISK);
-
         // sleep for the threshold time, so that the cache cleaner can clean the cache
         Instant end = Instant.now();
         long elapsedTimeMillis = Duration.between(start, end).toMillis();
@@ -352,18 +277,8 @@ public void testDiskTierInvalidationByCacheCleanerPartOfDiskTier() throws Except
         long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 5_000;
         Thread.sleep(sleepTime);
 
-        // fetch the stats again
-        requestCacheStats = client.admin()
-            .indices()
-            .prepareStats("index")
-            .setRequestCache(true)
-            .get()
-            .getTotal()
-            .getRequestCache();
-
         // make sure we have 5 entries in disk.
-        entries = requestCacheStats.getEntries(TierType.DISK);
-        assertEquals(5, entries);
+        IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 5, TierType.DISK);
     }
 
     private long getCacheSizeBytes(Client client, String index, TierType tierType) {

From ae0b47118dafccfeb51a10ad54268d3a6459e4dc Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Thu, 14 Dec 2023 17:19:14 -0800
Subject: [PATCH 117/120] update print statement

---
 .../org/opensearch/indices/IndicesRequestCacheDiskTierIT.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index f85b06ceeac44..475121d0967f7 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -266,7 +266,7 @@ public void testDiskTierInvalidationByCacheCleanerPartOfDiskTier() throws Except
         client().prepareIndex("index").setId("1").setSource("k", "good bye");
         ensureSearchable("index");
 
-        for (int i = 0; i < 6; i++) { // index 5 items with the new readerCacheKeyId
+        for (int i = 0; i < 6; i++) { // index 6 items with the new readerCacheKeyId
             client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
         }
 

From fce110d94095f1c107d1cd6f911a62c5642f6764 Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Thu, 14 Dec 2023 17:19:32 -0800
Subject: [PATCH 118/120] edit sleep time

---
 .../org/opensearch/indices/IndicesRequestCacheDiskTierIT.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
index 475121d0967f7..77e299497c9a5 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -211,7 +211,7 @@ public void testDiskTierInvalidationByCacheCleanerEntireDiskTier() throws Except
         Instant end = Instant.now();
         long elapsedTimeMillis = Duration.between(start, end).toMillis();
         // if this test is flaky, increase the sleep time.
-        long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 1_000;
+        long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 1_500;
         Thread.sleep(sleepTime);
 
         // by now cache cleaner would have run and cleaned up stale keys
@@ -274,7 +274,7 @@ public void testDiskTierInvalidationByCacheCleanerPartOfDiskTier() throws Except
         Instant end = Instant.now();
         long elapsedTimeMillis = Duration.between(start, end).toMillis();
         // if this test is flaky, increase the sleep time.
-        long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 5_000;
+        long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 1_500;
         Thread.sleep(sleepTime);
 
         // make sure we have 5 entries in disk.

From 7d2ea2efa07a71f6e821724f1bfeb5339f4f1851 Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Fri, 15 Dec 2023 12:15:41 -0800
Subject: [PATCH 119/120] changed setting paths from index to indices for
 non-index level settings

---
 .../org/opensearch/indices/IndicesRequestCache.java   |  8 ++++----
 .../tier/keystore/RBMIntKeyLookupStoreTests.java      | 11 +++++++++++
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 77c0662445f96..0457412cd7f22 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -128,27 +128,27 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
      */
 
     public static final Setting<Boolean> INDICES_CACHE_DISK_TIER_ENABLED = Setting.boolSetting(
-        "index.requests.cache.tiered.disk.enable",
+        "indices.requests.cache.tiered.disk.enable",
         false,
         Property.Dynamic,
         Property.NodeScope
     );
 
     public static final Setting<ByteSizeValue> INDICES_CACHE_DISK_TIER_SIZE = Setting.memorySizeSetting(
-        "index.requests.cache.tiered.disk.size",
+        "indices.requests.cache.tiered.disk.size",
         new ByteSizeValue(100, ByteSizeUnit.MB),
         Property.NodeScope
     ); // TODO: This is 1% of the minimum EBS size for most EC2 instances. In future set to 1% of actual disk size if possible
 
     public static final Setting<Double> INDICES_CACHE_DISK_STALE_KEY_THRESHOLD = Setting.doubleSetting(
-        "index.requests.cache.tiered.disk.stale_cleanup_threshold",
+        "indices.requests.cache.tiered.disk.stale_cleanup_threshold",
         0.5,
         Property.Dynamic,
         Property.NodeScope
     );
 
     public static final Setting<ByteSizeValue> INDICES_CACHE_KEYSTORE_SIZE = Setting.memorySizeSetting(
-        "index.requests.cache.tiered.disk.keystore.size",
+        "indices.requests.cache.tiered.disk.keystore.size",
         "0.05%", // 5% of INDICES_CACHE_QUERY_SIZE
         Property.Dynamic,
         Property.NodeScope
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
index f1c1caf11676b..1a76dd7ef2e89 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java
@@ -54,6 +54,17 @@ public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase {
     final int BYTES_IN_MB = 1048576;
     ClusterSettings DUMMY_CLUSTER_SETTINGS = new ClusterSettings(Settings.EMPTY, Set.of(RBMIntKeyLookupStore.INDICES_CACHE_KEYSTORE_SIZE));
 
+    public void testQuickSizeTest() {
+        int numToAdd = 1000000;
+        RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(DUMMY_CLUSTER_SETTINGS);
+        Random rand = Randomness.get();
+        for (int i = 0; i < numToAdd; i++) {
+            kls.add(rand.nextInt());
+        }
+        System.out.println(kls.getMemorySizeInBytes());
+        System.out.println(kls.isFull());
+    }
+
     public void testInit() {
         long memCap = 100 * BYTES_IN_MB;
         RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(memCap, DUMMY_CLUSTER_SETTINGS);

From 49c9bbddffe5d996b1e927f6ce6a4c2aaed1cb2a Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <petealft@amazon.com>
Date: Mon, 18 Dec 2023 14:13:18 -0800
Subject: [PATCH 120/120] Addressed Kiran's comments

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
---
 distribution/src/config/opensearch.yml        |  9 -----
 .../common/cache/tier/DiskTierProvider.java   |  2 +-
 .../cache/tier/EhCacheDiskCachingTier.java    | 33 +------------------
 .../TieredCacheSpilloverStrategyService.java  |  4 +--
 .../tier/keystore/RBMIntKeyLookupStore.java   | 10 +++++-
 .../common/settings/ClusterSettings.java      |  1 -
 .../indices/IndicesRequestCache.java          | 16 ++-------
 .../tier/EhCacheDiskCachingTierTests.java     | 13 --------
 8 files changed, 14 insertions(+), 74 deletions(-)

diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml
index 72c81a5d956ee..78d719edd599a 100644
--- a/distribution/src/config/opensearch.yml
+++ b/distribution/src/config/opensearch.yml
@@ -131,12 +131,3 @@ ${path.logs}
 #
 # Gates the tiered caching feature. This feature adds a disk tier to the IndicesRequestCache.
 # opensearch.experimental.feature.tiered_caching.enabled: false
-#
-# Settings related to tiered caching:
-#index.requests.cache.tiered.disk.keystore.size: 0.05%
-#index.requests.cache.tiered.disk.stale_cleanup_threshold: 0.5
-#indices.requests.cache.tiered.disk.ehcache.min_threads: 2
-#indices.requests.cache.tiered.disk.ehcache.max_threads: 2
-#indices.requests.cache.tiered.disk.ehcache.write_concurrency: 2
-#indices.requests.cache.tiered.disk.ehcache.segments: 16
-#
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java
index a98a728200bef..ebb68bbc2ae63 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java
@@ -12,5 +12,5 @@
  * An interface that provides new instances of a DiskTier.
  */
 public interface DiskTierProvider<K, V> {
-    DiskCachingTier<K, V> getDiskTier();
+    DiskCachingTier<K, V> createNewDiskTier();
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index a330d99d5b589..afdd7ea42d768 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -52,14 +52,9 @@
  * @param <V> The value type of cache entries
  */
 public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
-    public static final Setting<Double> REQUEST_CACHE_DISK_STALE_KEY_THRESHOLD = Setting.doubleSetting(
-        "indices.requests.cache.tiered.disk.stale_cleanup_threshold",
-        0.5,
-        Property.Dynamic,
-        Property.NodeScope
-    );
 
     // Ehcache disk write minimum threads for its pool
+    // All number values in setting constructors are default value, min value, and max value
     public static final Setting<Integer> REQUEST_CACHE_DISK_MIN_THREADS = Setting.intSetting(
         "indices.requests.cache.tiered.disk.ehcache.min_threads",
         2,
@@ -116,8 +111,6 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
     private final EhCacheEventListener<K, V> ehCacheEventListener;
 
     private final String threadPoolAlias;
-
-    private final Settings settings;
     private final ClusterSettings clusterSettings;
 
     private CounterMetric count = new CounterMetric();
@@ -147,9 +140,7 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         } else {
             this.threadPoolAlias = builder.threadPoolAlias;
         }
-        this.settings = Objects.requireNonNull(builder.settings, "Settings objects shouldn't be null");
         this.clusterSettings = Objects.requireNonNull(builder.clusterSettings, "ClusterSettings object shouldn't be null");
-        Objects.requireNonNull(builder.settingPrefix, "Setting prefix shouldn't be null");
 
         // In test cases, there might be leftover cache managers and caches hanging around, from nodes created in the test case setup
         // Destroy them before recreating them
@@ -158,7 +149,6 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
 
         this.keystore = new RBMIntKeyLookupStore(clusterSettings);
-        clusterSettings.addSettingsUpdateConsumer(REQUEST_CACHE_DISK_STALE_KEY_THRESHOLD, this::setStaleKeyThreshold);
     }
 
     private PersistentCacheManager buildCacheManager() {
@@ -311,11 +301,6 @@ public void close() {
         } catch (NullPointerException ignored) {} // Another test node has already destroyed the cache manager
     }
 
-    private void setStaleKeyThreshold(double newThreshold) {
-        return;
-        // TODO: Fill in once Kiran's code is merged
-    }
-
     /**
      * Wrapper over Ehcache original listener to listen to desired events and notify desired subscribers.
      * @param <K> Type of key
@@ -471,19 +456,14 @@ public static class Builder<K, V> {
         private String storagePath;
 
         private String threadPoolAlias;
-
-        private Settings settings;
         private ClusterSettings clusterSettings;
 
         private String diskCacheAlias;
 
-        private String settingPrefix;
-
         // Provides capability to make ehCache event listener to run in sync mode. Used for testing too.
         private boolean isEventListenerModeSync;
         private Serializer<K, byte[]> keySerializer;
         private Serializer<V, byte[]> valueSerializer;
-        private long keystoreMaxWeightInBytes = 0;
 
         public Builder() {}
 
@@ -520,22 +500,11 @@ public EhCacheDiskCachingTier.Builder<K, V> setThreadPoolAlias(String threadPool
             return this;
         }
 
-        public EhCacheDiskCachingTier.Builder<K, V> setSettings(Settings settings) {
-            this.settings = settings;
-            return this;
-        }
-
         public EhCacheDiskCachingTier.Builder<K, V> setDiskCacheAlias(String diskCacheAlias) {
             this.diskCacheAlias = diskCacheAlias;
             return this;
         }
 
-        public EhCacheDiskCachingTier.Builder<K, V> setSettingPrefix(String settingPrefix) {
-            // TODO: Do some basic validation. So that it doesn't end with "." etc.
-            this.settingPrefix = settingPrefix;
-            return this;
-        }
-
         public EhCacheDiskCachingTier.Builder<K, V> setIsEventListenerModeSync(boolean isEventListenerModeSync) {
             this.isEventListenerModeSync = isEventListenerModeSync;
             return this;
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 0d6996daf2e9c..a403f4d729497 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -12,10 +12,8 @@
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.indices.IndicesRequestCache;
-import org.opensearch.indices.IndicesService;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -256,7 +254,7 @@ private void toggleDiskTierEnabled(boolean enableDiskTier) {
                 enableInactiveDiskTier();
             } else {
                 // There is no disk tier to reactivate, create a new one now.
-                DiskCachingTier<K, V> newTier = diskTierProvider.getDiskTier();
+                DiskCachingTier<K, V> newTier = diskTierProvider.createNewDiskTier();
                 addNewDiskTier(newTier);
             }
         } else if (!enableDiskTier && diskTierInUse) {
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
index 5027813604042..a9734343fa762 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -92,7 +92,7 @@ public int getValue() {
     protected final Lock readLock = lock.readLock();
     protected final Lock writeLock = lock.writeLock();
     private long mostRecentByteEstimate;
-    public static final int REFRESH_SIZE_EST_INTERVAL = 10000;
+    static final int REFRESH_SIZE_EST_INTERVAL = 10_000;
     // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms,
     // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store
     // in terms of storage impact
@@ -319,11 +319,19 @@ static double getRBMSizeMultiplier(int numEntries, int modulo) {
         return 1;
     }
 
+    /**
+     * Return the most recent memory size estimate, without updating it.
+     * @return the size estimate (bytes)
+     */
     @Override
     public long getMemorySizeInBytes() {
         return mostRecentByteEstimate;
     }
 
+    /**
+     * Calculate a new memory size estimate. This is somewhat expensive so we don't call this every time we run get().
+     * @return a new size estimate (bytes)
+     */
     private long computeMemorySizeInBytes() {
         double multiplier = getRBMSizeMultiplier((int) stats.size.count(), modulo);
         return (long) (rbm.getSizeInBytes() * multiplier);
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index c9e9d09b1fa27..a1d7f2f4154fb 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -725,7 +725,6 @@ public void apply(Settings value, Settings current, Settings previous) {
             IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED,
             IndicesRequestCache.INDICES_CACHE_DISK_TIER_SIZE,
             RBMIntKeyLookupStore.INDICES_CACHE_KEYSTORE_SIZE,
-            EhCacheDiskCachingTier.REQUEST_CACHE_DISK_STALE_KEY_THRESHOLD,
             EhCacheDiskCachingTier.REQUEST_CACHE_DISK_MIN_THREADS,
             EhCacheDiskCachingTier.REQUEST_CACHE_DISK_MAX_THREADS,
             EhCacheDiskCachingTier.REQUEST_CACHE_DISK_WRITE_CONCURRENCY,
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 0457412cd7f22..416810b73a6e1 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -74,9 +74,7 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.Function;
@@ -140,13 +138,6 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
         Property.NodeScope
     ); // TODO: This is 1% of the minimum EBS size for most EC2 instances. In future set to 1% of actual disk size if possible
 
-    public static final Setting<Double> INDICES_CACHE_DISK_STALE_KEY_THRESHOLD = Setting.doubleSetting(
-        "indices.requests.cache.tiered.disk.stale_cleanup_threshold",
-        0.5,
-        Property.Dynamic,
-        Property.NodeScope
-    );
-
     public static final Setting<ByteSizeValue> INDICES_CACHE_KEYSTORE_SIZE = Setting.memorySizeSetting(
         "indices.requests.cache.tiered.disk.keystore.size",
         "0.05%", // 5% of INDICES_CACHE_QUERY_SIZE
@@ -193,7 +184,7 @@ public final class IndicesRequestCache implements TieredCacheEventListener<Indic
 
         if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
             if (clusterSettings.get(INDICES_CACHE_DISK_TIER_ENABLED)) {
-                EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = getDiskTier();
+                EhCacheDiskCachingTier<Key, BytesReference> ehcacheDiskTier = createNewDiskTier();
                 tieredCacheServiceBuilder.setOnDiskCachingTier(ehcacheDiskTier);
             }
             // if the disk tier dynamic setting is disabled, still provide the necessary things to turn it on
@@ -503,9 +494,8 @@ int numRegisteredCloseListeners() { // for testing
      * @return A new disk tier instance
      */
     @Override
-    public EhCacheDiskCachingTier<Key, BytesReference> getDiskTier() {
+    public EhCacheDiskCachingTier<Key, BytesReference> createNewDiskTier() {
         assert FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING);
-        String SETTING_PREFIX = "indices.request.cache";
         long CACHE_SIZE_IN_BYTES = INDICES_CACHE_DISK_TIER_SIZE.get(settings).getBytes();
         String STORAGE_PATH = indicesService.getNodePaths()[0].indicesPath.toString() + "/request_cache";
 
@@ -513,12 +503,10 @@ public EhCacheDiskCachingTier<Key, BytesReference> getDiskTier() {
             .setKeyType(Key.class)
             .setValueType(BytesReference.class)
             .setExpireAfterAccess(TimeValue.MAX_VALUE) // TODO: Is this meant to be the same as IRC expire or different?
-            .setSettings(settings)
             .setClusterSettings(clusterSettings)
             .setThreadPoolAlias("ehcacheThreadpool")
             .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
             .setStoragePath(STORAGE_PATH)
-            .setSettingPrefix(SETTING_PREFIX)
             .setKeySerializer(new IRCKeyWriteableSerializer(this))
             .setValueSerializer(new BytesReferenceSerializer())
             .build();
diff --git a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
index 27b5c6db74549..b5cdd6aefed54 100644
--- a/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTierTests.java
@@ -37,7 +37,6 @@
 public class EhCacheDiskCachingTierTests extends OpenSearchSingleNodeTestCase {
 
     private static final int CACHE_SIZE_IN_BYTES = 1024 * 101;
-    private static final String SETTING_PREFIX = "indices.request.cache";
 
     public void testBasicGetAndPut() throws IOException {
         Settings settings = Settings.builder().build();
@@ -46,12 +45,10 @@ public void testBasicGetAndPut() throws IOException {
                 .setKeyType(String.class)
                 .setValueType(String.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
-                .setSettings(settings)
                 .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
-                .setSettingPrefix(SETTING_PREFIX)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
                 .build();
@@ -82,12 +79,10 @@ public void testBasicGetAndPutBytesReference() throws Exception {
                 BytesReference>().setKeyType(String.class)
                 .setValueType(BytesReference.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
-                .setSettings(settings)
                 .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES * 2) // bigger so no evictions happen
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
-                .setSettingPrefix(SETTING_PREFIX)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new BytesReferenceSerializer())
                 .build();
@@ -118,12 +113,10 @@ public void testConcurrentPut() throws Exception {
                 .setKeyType(String.class)
                 .setValueType(String.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
-                .setSettings(settings)
                 .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
-                .setSettingPrefix(SETTING_PREFIX)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
                 .build();
@@ -162,12 +155,10 @@ public void testEhcacheParallelGets() throws Exception {
                 .setKeyType(String.class)
                 .setValueType(String.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
-                .setSettings(settings)
                 .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
-                .setSettingPrefix(SETTING_PREFIX)
                 .setIsEventListenerModeSync(true) // For accurate count
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
@@ -208,11 +199,9 @@ public void testEhcacheKeyIterator() throws Exception {
                 .setKeyType(String.class)
                 .setValueType(String.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
-                .setSettings(settings)
                 .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
-                .setSettingPrefix(SETTING_PREFIX)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
@@ -245,11 +234,9 @@ public void testCompute() throws Exception {
                 .setKeyType(String.class)
                 .setValueType(String.class)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
-                .setSettings(settings)
                 .setClusterSettings(getClusterSettings())
                 .setThreadPoolAlias("ehcacheTest")
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
-                .setSettingPrefix(SETTING_PREFIX)
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())