Merge remote-tracking branch 'origin/main' into HEAD

Signed-off-by: Peter Nied <[email protected]>
opensearch-project · Jan 9, 2024 · 6fccb44 · 6fccb44
2 parents 3ca14d7 + a1e4602
commit 6fccb44
Show file tree

Hide file tree

Showing 162 changed files with 2,943 additions and 373 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_template.yml b/.github/ISSUE_TEMPLATE/bug_template.yml
@@ -15,7 +15,7 @@ body:
       description: Choose a specific OpenSearch component your bug belongs to.  If you are unsure which to select or if the component is not present, select "Other".
       multiple: false
       options:
-        - Other
+        - # Empty first option to force selection
         - Build
         - Clients
         - Cluster Manager
@@ -24,6 +24,7 @@ body:
         - Indexing:Replication
         - Indexing
         - Libraries
+        - Other
         - Plugins
         - Search:Aggregations
         - Search:Performance

diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml
@@ -22,7 +22,7 @@ body:
       description: Choose a specific OpenSearch component your feature request belongs to.  If you are unsure of which component to select or if the component is not present, select "Other".
       multiple: false
       options:
-        - Other
+        - # Empty first option to force selection
         - Build
         - Clients
         - Cluster Manager
@@ -31,6 +31,7 @@ body:
         - Indexing:Replication
         - Indexing
         - Libraries
+        - Other
         - Plugins
         - Search:Aggregations
         - Search:Performance

diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml
@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: lychee Link Checker
         id: lychee
-        uses: lycheeverse/lychee-action@v1.8.0
+        uses: lycheeverse/lychee-action@v1.9.0
         with:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -107,6 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
 - [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672))
+- Enable must_exist parameter for update aliases API ([#11210](https://github.com/opensearch-project/OpenSearch/pull/11210))
 - Add back half_float BKD based sort query optimization ([#11024](https://github.com/opensearch-project/OpenSearch/pull/11024))
 - Request level coordinator slow logs ([#10650](https://github.com/opensearch-project/OpenSearch/pull/10650))
 - Add template snippets support for field and target_field in KV ingest processor ([#10040](https://github.com/opensearch-project/OpenSearch/pull/10040))
@@ -120,11 +121,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add additional handling in SearchTemplateRequest when simulate is set to true ([#11591](https://github.com/opensearch-project/OpenSearch/pull/11591))
 - Introduce cluster level setting `cluster.index.restrict.replication.type` to prevent replication type setting override during index creations([#11583](https://github.com/opensearch-project/OpenSearch/pull/11583))
 - Add match_only_text field that is optimized for storage by trading off positional queries performance ([#6836](https://github.com/opensearch-project/OpenSearch/pull/11039))
+- Introduce new feature flag "WRITEABLE_REMOTE_INDEX" to gate the writeable remote index functionality ([#11717](https://github.com/opensearch-project/OpenSearch/pull/11170))
 
 ### Dependencies
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
 - Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
-- Bump `commons-io:commons-io` from 2.13.0 to 2.15.1 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294), [#11001](https://github.com/opensearch-project/OpenSearch/pull/11001), [#11002](https://github.com/opensearch-project/OpenSearch/pull/11002), [#11446](https://github.com/opensearch-project/OpenSearch/pull/11446), [#11554](https://github.com/opensearch-project/OpenSearch/pull/11554), [#11560](https://github.com/opensearch-project/OpenSearch/pull/11560))
+- Bump `commons-io:commons-io` from 2.13.0 to 2.15.1 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294), [#11001](https://github.com/opensearch-project/OpenSearch/pull/11001), [#11002](https://github.com/opensearch-project/OpenSearch/pull/11002), [#11446](https://github.com/opensearch-project/OpenSearch/pull/11446), [#11554](https://github.com/opensearch-project/OpenSearch/pull/11554), [#11560](https://github.com/opensearch-project/OpenSearch/pull/11560), [#11796](https://github.com/opensearch-project/OpenSearch/pull/11796))
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
 - Bump `com.netflix.nebula.ospackage-base` from 11.4.0 to 11.6.0 ([#10295](https://github.com/opensearch-project/OpenSearch/pull/10295), [#11630](https://github.com/opensearch-project/OpenSearch/pull/11630))
 - Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
@@ -138,7 +140,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `stefanzweifel/git-auto-commit-action` from 4 to 5 ([#11171](https://github.com/opensearch-project/OpenSearch/pull/11171))
 - Bump `actions/github-script` from 6 to 7 ([#11271](https://github.com/opensearch-project/OpenSearch/pull/11271))
 - Bump `jackson` and `jackson_databind` from 2.15.2 to 2.16.0 ([#11273](https://github.com/opensearch-project/OpenSearch/pull/11273))
-- Bump `netty` from 4.1.100.Final to 4.1.101.Final ([#11294](https://github.com/opensearch-project/OpenSearch/pull/11294))
+- Bump `netty` from 4.1.100.Final to 4.1.104.Final ([#11294](https://github.com/opensearch-project/OpenSearch/pull/11294), [#11775](https://github.com/opensearch-project/OpenSearch/pull/11775))
 - Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.16.12 to 0.17.6 ([#10163](https://github.com/opensearch-project/OpenSearch/pull/10163), [#11692](https://github.com/opensearch-project/OpenSearch/pull/11692))
 - Bump `com.squareup.okhttp3:okhttp` from 4.11.0 to 4.12.0 ([#10861](https://github.com/opensearch-project/OpenSearch/pull/10861))
 - Bump `org.apache.commons:commons-text` from 1.10.0 to 1.11.0 ([#11344](https://github.com/opensearch-project/OpenSearch/pull/11344))
@@ -158,29 +160,34 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.maxmind.geoip2:geoip2` from 4.1.0 to 4.2.0 ([#11559](https://github.com/opensearch-project/OpenSearch/pull/11559))
 - Bump `org.apache.commons:commons-lang3` from 3.13.0 to 3.14.0 ([#11691](https://github.com/opensearch-project/OpenSearch/pull/11691))
 - Bump `com.maxmind.db:maxmind-db` from 3.0.0 to 3.1.0 ([#11693](https://github.com/opensearch-project/OpenSearch/pull/11693))
+- Bump `net.java.dev.jna:jna` from 5.13.0 to 5.14.0 ([#11798](https://github.com/opensearch-project/OpenSearch/pull/11798))
+- Bump `lycheeverse/lychee-action` from 1.8.0 to 1.9.0 ([#11795](https://github.com/opensearch-project/OpenSearch/pull/11795))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
 - Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
-- Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
+- Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562)), ([#11751](https://github.com/opensearch-project/OpenSearch/pull/11751))
 - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107))
 - Search pipelines now support asynchronous request and response processors to avoid blocking on a transport thread ([#10598](https://github.com/opensearch-project/OpenSearch/pull/10598))
 - [Remote Store] Add Remote Store backpressure rejection stats to `_nodes/stats` ([#10524](https://github.com/opensearch-project/OpenSearch/pull/10524))
 - [BUG] Fix java.lang.SecurityException in repository-gcs plugin ([#10642](https://github.com/opensearch-project/OpenSearch/pull/10642))
 - Add telemetry tracer/metric enable flag and integ test. ([#10395](https://github.com/opensearch-project/OpenSearch/pull/10395))
 - Performance improvement for Datetime field caching ([#4558](https://github.com/opensearch-project/OpenSearch/issues/4558))
 - Add instrumentation for indexing in transport bulk action and transport shard bulk action. ([#10273](https://github.com/opensearch-project/OpenSearch/pull/10273))
-- Disallow removing some metadata fields by remove ingest processor ([#10895](https://github.com/opensearch-project/OpenSearch/pull/10895))
+- Disallow removing some metadata fields by remove ingest processor ([#10895](https://github.com/opensearch-project/OpenSearch/pull/10895), [#11607](https://github.com/opensearch-project/OpenSearch/pull/11607))
 - Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
 - Refactor common parts from the Rounding class into a separate 'round' package ([#11023](https://github.com/opensearch-project/OpenSearch/issues/11023))
 - Performance improvement for date histogram aggregations without sub-aggregations ([#11083](https://github.com/opensearch-project/OpenSearch/pull/11083))
 - Disable concurrent aggs for Diversified Sampler and Sampler aggs ([#11087](https://github.com/opensearch-project/OpenSearch/issues/11087))
 - Made leader/follower check timeout setting dynamic ([#10528](https://github.com/opensearch-project/OpenSearch/pull/10528))
+- Improved performance of numeric exact-match queries ([#11209](https://github.com/opensearch-project/OpenSearch/pull/11209))
 - Change error message when per shard document limit is breached ([#11312](https://github.com/opensearch-project/OpenSearch/pull/11312))
 - Improve boolean parsing performance ([#11308](https://github.com/opensearch-project/OpenSearch/pull/11308))
 - Interpret byte array as primitive using VarHandles ([#11362](https://github.com/opensearch-project/OpenSearch/pull/11362))
 - Automatically add scheme to discovery.ec2.endpoint ([#11512](https://github.com/opensearch-project/OpenSearch/pull/11512))
 - Restore support for Java 8 for RestClient ([#11562](https://github.com/opensearch-project/OpenSearch/pull/11562))
+- Add deleted doc count in _cat/shards ([#11678](https://github.com/opensearch-project/OpenSearch/pull/11678))
+- Capture information for additional query types and aggregation types ([#11582](https://github.com/opensearch-project/OpenSearch/pull/11582))
 
 ### Deprecated
 
@@ -189,7 +196,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Fixed
 - Fix failure in dissect ingest processor parsing empty brackets ([#9225](https://github.com/opensearch-project/OpenSearch/pull/9255))
-- Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101))
+- Fix `class_cast_exception` when passing int to `_version` and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101))
 - Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))
 - Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496))
 - Fix passing wrong parameter when calling newConfigurationException() in DotExpanderProcessor ([#10737](https://github.com/opensearch-project/OpenSearch/pull/10737))
@@ -201,6 +208,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix the issue with DefaultSpanScope restoring wrong span in the TracerContextStorage upon detach ([#11316](https://github.com/opensearch-project/OpenSearch/issues/11316))
 - Remove shadowJar from `lang-painless` module publication ([#11369](https://github.com/opensearch-project/OpenSearch/issues/11369))
 - Fix remote shards balancer and remove unused variables ([#11167](https://github.com/opensearch-project/OpenSearch/pull/11167))
+- Fix parsing of flat object fields with dots in keys ([#11425](https://github.com/opensearch-project/OpenSearch/pull/11425))
 - Fix bug where replication lag grows post primary relocation ([#11238](https://github.com/opensearch-project/OpenSearch/pull/11238))
 - Fix for stuck update action in a bulk with `retry_on_conflict` property ([#11152](https://github.com/opensearch-project/OpenSearch/issues/11152))
 - Fix template setting override for replication type ([#11417](https://github.com/opensearch-project/OpenSearch/pull/11417))

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -110,7 +110,7 @@ dependencies {
   api 'com.netflix.nebula:gradle-info-plugin:12.1.6'
   api 'org.apache.rat:apache-rat:0.15'
   api 'commons-io:commons-io:2.15.1'
-  api "net.java.dev.jna:jna:5.13.0"
+  api "net.java.dev.jna:jna:5.14.0"
   api 'com.github.johnrengelman:shadow:8.1.1'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -26,7 +26,7 @@ jakarta_annotation = 1.3.5
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.101.Final
+netty             = 4.1.104.Final
 joda              = 2.12.2
 
 # project reactor

diff --git a/...yamlRestTest/resources/rest-api-spec/test/search.query/11_match_field_match_only_text.yml b/...yamlRestTest/resources/rest-api-spec/test/search.query/11_match_field_match_only_text.yml
@@ -2,8 +2,8 @@
 
 "match query with stacked stems":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   # Tests the match query stemmed tokens are "stacked" on top of the unstemmed
   # versions in the same position.
   - do:

diff --git a/...tTest/resources/rest-api-spec/test/search.query/20_ngram_search_field_match_only_text.yml b/...tTest/resources/rest-api-spec/test/search.query/20_ngram_search_field_match_only_text.yml
@@ -1,7 +1,7 @@
 "ngram search":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       indices.create:
         index: test
@@ -45,8 +45,8 @@
 ---
 "testNGramCopyField":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       indices.create:
         index: test

diff --git a/...resources/rest-api-spec/test/search.query/30_ngram_highligthing_field_match_only_text.yml b/...resources/rest-api-spec/test/search.query/30_ngram_highligthing_field_match_only_text.yml
@@ -1,7 +1,7 @@
 "ngram highlighting":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       indices.create:
         index: test

diff --git a/...tTest/resources/rest-api-spec/test/search.query/40_query_string_field_match_only_text.yml b/...tTest/resources/rest-api-spec/test/search.query/40_query_string_field_match_only_text.yml
@@ -1,8 +1,8 @@
 ---
 "Test query string with snowball":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       indices.create:
           index:  test

diff --git a/...pi-spec/test/search.query/41_query_string_with_default_analyzer_field_match_only_text.yml b/...pi-spec/test/search.query/41_query_string_with_default_analyzer_field_match_only_text.yml
@@ -1,8 +1,8 @@
 ---
 "Test default search analyzer is applied":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       indices.create:
         index: test

diff --git a/...ources/rest-api-spec/test/search.query/50_queries_with_synonyms_field_match_only_text.yml b/...ources/rest-api-spec/test/search.query/50_queries_with_synonyms_field_match_only_text.yml
@@ -1,8 +1,8 @@
 ---
 "Test common terms query with stacked tokens":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
       features: "allowed_warnings"
 
   - do:
@@ -247,8 +247,8 @@
 ---
 "Test match query with synonyms - see #3881 for extensive description of the issue":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       indices.create:
           index: test

diff --git a/...Test/resources/rest-api-spec/test/search.query/60_synonym_graph_field_match_only_text.yml b/...Test/resources/rest-api-spec/test/search.query/60_synonym_graph_field_match_only_text.yml
@@ -1,7 +1,7 @@
 setup:
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       indices.create:
         index: test

diff --git a/...RestTest/resources/rest-api-spec/test/search.query/70_intervals_field_match_only_text.yml b/...RestTest/resources/rest-api-spec/test/search.query/70_intervals_field_match_only_text.yml
@@ -1,8 +1,8 @@
 # integration tests for intervals queries using analyzers
 setup:
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       indices.create:
         index:  test
@@ -26,8 +26,8 @@ setup:
 ---
 "Test use_field":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       catch: bad_request
       search:

diff --git a/...lRestTest/resources/rest-api-spec/test/search.suggest/20_phrase_field_match_only_text.yml b/...lRestTest/resources/rest-api-spec/test/search.suggest/20_phrase_field_match_only_text.yml
@@ -2,8 +2,8 @@
 
 setup:
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       indices.create:
         index: test
@@ -122,8 +122,8 @@ setup:
 ---
 "breaks ties by sorting terms":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   # This runs the suggester without bigrams so we can be sure of the sort order
   - do:
       search:
@@ -181,8 +181,8 @@ setup:
 ---
 "doesn't fail when asked to run on a field without unigrams when force_unigrams=false":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       search:
         rest_total_hits_as_int: true
@@ -213,8 +213,8 @@ setup:
 ---
 "reverse suggestions":
   - skip:
-      version: " - 2.99.99"
-      reason: "match_only_text was added in 3.0"
+      version: " - 2.11.99"
+      reason: "match_only_text was added in 2.12"
   - do:
       search:
         rest_total_hits_as_int: true

diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RemoveProcessor.java
@@ -88,16 +88,18 @@ public IngestDocument execute(IngestDocument document) {
                 throw new IllegalArgumentException("cannot remove metadata field [" + path + "]");
             }
             // removing _id is disallowed when there's an external version specified in the request
-            String versionType = document.getFieldValue(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), String.class);
             if (path.equals(IngestDocument.Metadata.ID.getFieldName())
-                && !Objects.equals(versionType, VersionType.toString(VersionType.INTERNAL))) {
-                Long version = document.getFieldValue(IngestDocument.Metadata.VERSION.getFieldName(), Long.class);
-                throw new IllegalArgumentException(
-                    "cannot remove metadata field [_id] when specifying external version for the document, version: "
-                        + version
-                        + ", version_type: "
-                        + versionType
-                );
+                && document.hasField(IngestDocument.Metadata.VERSION_TYPE.getFieldName())) {
+                String versionType = document.getFieldValue(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), String.class);
+                if (!Objects.equals(versionType, VersionType.toString(VersionType.INTERNAL))) {
+                    Long version = document.getFieldValue(IngestDocument.Metadata.VERSION.getFieldName(), Long.class, true);
+                    throw new IllegalArgumentException(
+                        "cannot remove metadata field [_id] when specifying external version for the document, version: "
+                            + version
+                            + ", version_type: "
+                            + versionType
+                    );
+                }
             }
             document.removeField(path);
         });