From 8b476796888038614b9b79336badb50264bd199d Mon Sep 17 00:00:00 2001 From: Jackie Han Date: Mon, 28 Aug 2023 11:37:21 -0700 Subject: [PATCH] Revert "Enforce DOCUMENT Replication for AD Indices and Adjust Primary Shards (#948)" This reverts commit bc1649922ef39619424fc16c64af5836e48a4a12. --- .../ad/indices/ADIndexManagement.java | 15 +++--------- .../indices/ForecastIndexManagement.java | 12 +++------- .../forecast/settings/ForecastSettings.java | 2 +- .../timeseries/indices/IndexManagement.java | 23 ++++--------------- 4 files changed, 11 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/opensearch/ad/indices/ADIndexManagement.java b/src/main/java/org/opensearch/ad/indices/ADIndexManagement.java index 95ce25faa..d0a40ecd8 100644 --- a/src/main/java/org/opensearch/ad/indices/ADIndexManagement.java +++ b/src/main/java/org/opensearch/ad/indices/ADIndexManagement.java @@ -19,8 +19,6 @@ import static org.opensearch.ad.settings.AnomalyDetectorSettings.ANOMALY_DETECTION_STATE_INDEX_MAPPING_FILE; import static org.opensearch.ad.settings.AnomalyDetectorSettings.ANOMALY_RESULTS_INDEX_MAPPING_FILE; import static org.opensearch.ad.settings.AnomalyDetectorSettings.CHECKPOINT_INDEX_MAPPING_FILE; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; -import static org.opensearch.indices.replication.common.ReplicationType.DOCUMENT; import java.io.IOException; import java.util.EnumMap; @@ -66,7 +64,7 @@ public class ADIndexManagement extends IndexManagement { * @param settings OS cluster setting * @param nodeFilter Used to filter eligible nodes to host AD indices * @param maxUpdateRunningTimes max number of retries to update index mapping and setting - * @throws IOException when failing to get mapping file + * @throws IOException */ public ADIndexManagement( Client client, @@ -197,10 +195,7 @@ public void initDefaultResultIndexDirectly(ActionListener a @Override public void initStateIndex(ActionListener actionListener) { try { - // AD indices need RAW (e.g., we want users to be able to consume AD results as soon as possible and send out an alert if - // anomalies found). - Settings replicationSettings = Settings.builder().put(SETTING_REPLICATION_TYPE, DOCUMENT.name()).build(); - CreateIndexRequest request = new CreateIndexRequest(ADCommonName.DETECTION_STATE_INDEX, replicationSettings) + CreateIndexRequest request = new CreateIndexRequest(ADCommonName.DETECTION_STATE_INDEX) .mapping(getStateMappings(), XContentType.JSON) .settings(settings); adminClient.indices().create(request, markMappingUpToDate(ADIndex.STATE, actionListener)); @@ -224,11 +219,7 @@ public void initCheckpointIndex(ActionListener actionListen } catch (IOException e) { throw new EndRunException("", "Cannot find checkpoint mapping file", true); } - // AD indices need RAW (e.g., we want users to be able to consume AD results as soon as possible and send out an alert if anomalies - // found). - Settings replicationSettings = Settings.builder().put(SETTING_REPLICATION_TYPE, DOCUMENT.name()).build(); - CreateIndexRequest request = new CreateIndexRequest(ADCommonName.CHECKPOINT_INDEX_NAME, replicationSettings) - .mapping(mapping, XContentType.JSON); + CreateIndexRequest request = new CreateIndexRequest(ADCommonName.CHECKPOINT_INDEX_NAME).mapping(mapping, XContentType.JSON); choosePrimaryShards(request, true); adminClient.indices().create(request, markMappingUpToDate(ADIndex.CHECKPOINT, actionListener)); } diff --git a/src/main/java/org/opensearch/forecast/indices/ForecastIndexManagement.java b/src/main/java/org/opensearch/forecast/indices/ForecastIndexManagement.java index f27aa749e..e7d3f3252 100644 --- a/src/main/java/org/opensearch/forecast/indices/ForecastIndexManagement.java +++ b/src/main/java/org/opensearch/forecast/indices/ForecastIndexManagement.java @@ -11,7 +11,6 @@ package org.opensearch.forecast.indices; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.forecast.constant.ForecastCommonName.DUMMY_FORECAST_RESULT_ID; import static org.opensearch.forecast.settings.ForecastSettings.FORECAST_CHECKPOINT_INDEX_MAPPING_FILE; import static org.opensearch.forecast.settings.ForecastSettings.FORECAST_MAX_PRIMARY_SHARDS; @@ -20,7 +19,6 @@ import static org.opensearch.forecast.settings.ForecastSettings.FORECAST_RESULT_HISTORY_RETENTION_PERIOD; import static org.opensearch.forecast.settings.ForecastSettings.FORECAST_RESULT_HISTORY_ROLLOVER_PERIOD; import static org.opensearch.forecast.settings.ForecastSettings.FORECAST_STATE_INDEX_MAPPING_FILE; -import static org.opensearch.indices.replication.common.ReplicationType.DOCUMENT; import java.io.IOException; import java.util.EnumMap; @@ -63,7 +61,7 @@ public class ForecastIndexManagement extends IndexManagement { * @param settings OS cluster setting * @param nodeFilter Used to filter eligible nodes to host forecast indices * @param maxUpdateRunningTimes max number of retries to update index mapping and setting - * @throws IOException when failing to get mapping file + * @throws IOException */ public ForecastIndexManagement( Client client, @@ -179,8 +177,7 @@ public boolean doesCheckpointIndexExist() { @Override public void initStateIndex(ActionListener actionListener) { try { - Settings replicationSettings = Settings.builder().put(SETTING_REPLICATION_TYPE, DOCUMENT.name()).build(); - CreateIndexRequest request = new CreateIndexRequest(ForecastCommonName.FORECAST_STATE_INDEX, replicationSettings) + CreateIndexRequest request = new CreateIndexRequest(ForecastCommonName.FORECAST_STATE_INDEX) .mapping(getStateMappings(), XContentType.JSON) .settings(settings); adminClient.indices().create(request, markMappingUpToDate(ForecastIndex.STATE, actionListener)); @@ -204,10 +201,7 @@ public void initCheckpointIndex(ActionListener actionListen } catch (IOException e) { throw new EndRunException("", "Cannot find checkpoint mapping file", true); } - // forecast indices need RAW (e.g., we want users to be able to consume forecast results as soon as - // possible and send out an alert if a threshold is breached). - Settings replicationSettings = Settings.builder().put(SETTING_REPLICATION_TYPE, DOCUMENT.name()).build(); - CreateIndexRequest request = new CreateIndexRequest(ForecastCommonName.FORECAST_CHECKPOINT_INDEX_NAME, replicationSettings) + CreateIndexRequest request = new CreateIndexRequest(ForecastCommonName.FORECAST_CHECKPOINT_INDEX_NAME) .mapping(mapping, XContentType.JSON); choosePrimaryShards(request, true); adminClient.indices().create(request, markMappingUpToDate(ForecastIndex.CHECKPOINT, actionListener)); diff --git a/src/main/java/org/opensearch/forecast/settings/ForecastSettings.java b/src/main/java/org/opensearch/forecast/settings/ForecastSettings.java index 6b4078ad4..8aeaeb6c3 100644 --- a/src/main/java/org/opensearch/forecast/settings/ForecastSettings.java +++ b/src/main/java/org/opensearch/forecast/settings/ForecastSettings.java @@ -122,7 +122,7 @@ public final class ForecastSettings { // max number of primary shards of a forecast index public static final Setting FORECAST_MAX_PRIMARY_SHARDS = Setting - .intSetting("plugins.forecast.max_primary_shards", 20, 0, 200, Setting.Property.NodeScope, Setting.Property.Dynamic); + .intSetting("plugins.forecast.max_primary_shards", 10, 0, 200, Setting.Property.NodeScope, Setting.Property.Dynamic); // saving checkpoint every 12 hours. // To support 1 million entities in 36 data nodes, each node has roughly 28K models. diff --git a/src/main/java/org/opensearch/timeseries/indices/IndexManagement.java b/src/main/java/org/opensearch/timeseries/indices/IndexManagement.java index 747f2bfac..36134c263 100644 --- a/src/main/java/org/opensearch/timeseries/indices/IndexManagement.java +++ b/src/main/java/org/opensearch/timeseries/indices/IndexManagement.java @@ -11,8 +11,6 @@ package org.opensearch.timeseries.indices; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; -import static org.opensearch.indices.replication.common.ReplicationType.DOCUMENT; import static org.opensearch.timeseries.constant.CommonMessages.CAN_NOT_FIND_RESULT_INDEX; import java.io.IOException; @@ -428,10 +426,7 @@ public void initConfigIndexIfAbsent(ActionListener actionLi * @throws IOException IOException from {@link IndexManagement#getConfigMappings} */ public void initConfigIndex(ActionListener actionListener) throws IOException { - // time series indices need RAW (e.g., we want users to be able to consume AD results as soon as possible - // and send out an alert if anomalies found). - Settings replicationSettings = Settings.builder().put(SETTING_REPLICATION_TYPE, DOCUMENT.name()).build(); - CreateIndexRequest request = new CreateIndexRequest(CommonName.CONFIG_INDEX, replicationSettings) + CreateIndexRequest request = new CreateIndexRequest(CommonName.CONFIG_INDEX) .mapping(getConfigMappings(), XContentType.JSON) .settings(settings); adminClient.indices().create(request, actionListener); @@ -482,11 +477,7 @@ public static String getJobMappings() throws IOException { */ public void initJobIndex(ActionListener actionListener) { try { - // time series indices need RAW (e.g., we want users to be able to consume AD results as soon as - // possible and send out an alert if anomalies found). - Settings replicationSettings = Settings.builder().put(SETTING_REPLICATION_TYPE, DOCUMENT.name()).build(); - CreateIndexRequest request = new CreateIndexRequest(CommonName.JOB_INDEX, replicationSettings) - .mapping(getJobMappings(), XContentType.JSON); + CreateIndexRequest request = new CreateIndexRequest(CommonName.JOB_INDEX).mapping(getJobMappings(), XContentType.JSON); request .settings( Settings @@ -937,10 +928,7 @@ protected void rolloverAndDeleteHistoryIndex( CreateIndexRequest createRequest = rollOverRequest.getCreateIndexRequest(); - // time series indices need RAW (e.g., we want users to be able to consume AD results as soon as possible - // and send out an alert if anomalies found). - Settings replicationSettings = Settings.builder().put(SETTING_REPLICATION_TYPE, DOCUMENT.name()).build(); - createRequest.index(rolloverIndexPattern).settings(replicationSettings).mapping(resultMapping, XContentType.JSON); + createRequest.index(rolloverIndexPattern).mapping(resultMapping, XContentType.JSON); choosePrimaryShards(createRequest, true); @@ -965,10 +953,7 @@ protected void initResultIndexDirectly( IndexType resultIndex, ActionListener actionListener ) { - // time series indices need RAW (e.g., we want users to be able to consume AD results as soon as possible - // and send out an alert if anomalies found). - Settings replicationSettings = Settings.builder().put(SETTING_REPLICATION_TYPE, DOCUMENT.name()).build(); - CreateIndexRequest request = new CreateIndexRequest(resultIndexName, replicationSettings).mapping(resultMapping, XContentType.JSON); + CreateIndexRequest request = new CreateIndexRequest(resultIndexName).mapping(resultMapping, XContentType.JSON); if (alias != null) { request.alias(new Alias(alias)); }