Merge branch 'main' into agg-per

Signed-off-by: Sandesh Kumar <[email protected]>

.github/ISSUE_TEMPLATE/meta.yml

@@ -23,6 +23,36 @@ body:
       placeholder: Ex. - [ ] https://github.com/opensearch-project/security/issues/3888 Add views to the cluster metadata schema
     validations:
       required: true
-
-
-
+  - type: dropdown
+    attributes:
+      label: Related component
+      description: Choose a specific OpenSearch component your project belongs to.  If you are unsure of which component to select or if the component is not present, select "Other".
+      multiple: false
+      options:
+        - # Empty first option to force selection
+        - Build
+        - Clients
+        - Cluster Manager
+        - Extensions
+        - Indexing:Performance
+        - Indexing:Replication
+        - Indexing
+        - Libraries
+        - Other
+        - Plugins
+        - Search:Aggregations
+        - Search:Performance
+        - Search:Query Capabilities
+        - Search:Query Insights
+        - Search:Relevance
+        - Search:Remote Search
+        - Search:Resiliency
+        - Search:Searchable Snapshots
+        - Search
+        - Storage:Durability
+        - Storage:Performance
+        - Storage:Remote
+        - Storage:Snapshots
+        - Storage
+    validations:
+      required: true

CHANGELOG.md

@@ -48,6 +48,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))
 - Bump OpenTelemetry from 1.30.1 to 1.31.0 ([#10617](https://github.com/opensearch-project/OpenSearch/pull/10617))
 - Bump OpenTelemetry from 1.31.0 to 1.32.0 and OpenTelemetry Semconv from 1.21.0-alpha to 1.23.1-alpha ([#11305](https://github.com/opensearch-project/OpenSearch/pull/11305))
+- Bump `org.bouncycastle:bcprov-jdk15to18` to `org.bouncycastle:bcprov-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
+- Bump `org.bouncycastle:bcmail-jdk15to18` to `org.bouncycastle:bcmail-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
+- Bump `org.bouncycastle:bcpkix-jdk15to18` to `org.bouncycastle:bcpkix-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -100,8 +103,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Dependencies
 - Bump `peter-evans/find-comment` from 2 to 3 ([#12288](https://github.com/opensearch-project/OpenSearch/pull/12288))
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.25.1 to 2.33.0 ([#12289](https://github.com/opensearch-project/OpenSearch/pull/12289))
+- Bump `com.squareup.okio:okio` from 3.7.0 to 3.8.0 ([#12290](https://github.com/opensearch-project/OpenSearch/pull/12290))
 
 ### Changed
+- Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499))
 - Improve string terms aggregation performance using Collector#setWeight ([#11643](https://github.com/opensearch-project/OpenSearch/pull/11643))
 
 ### Deprecated
@@ -110,6 +115,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Fixed
 - Fix for deserilization bug in weighted round-robin metadata ([#11679](https://github.com/opensearch-project/OpenSearch/pull/11679))
+- Add a system property to configure YamlParser codepoint limits ([#12298](https://github.com/opensearch-project/OpenSearch/pull/12298))
+- Prevent read beyond slice boundary in ByteArrayIndexInput ([#10481](https://github.com/opensearch-project/OpenSearch/issues/10481))
 
 ### Security
 

buildSrc/version.properties

@@ -50,7 +50,7 @@ reactivestreams   = 1.0.4
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
-bouncycastle=1.76
+bouncycastle=1.77
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.13.2

gradle/ide.gradle

@@ -82,6 +82,9 @@ if (System.getProperty('idea.active') == 'true') {
         runConfigurations {
           defaults(JUnit) {
             vmParameters = '-ea -Djava.locale.providers=SPI,COMPAT'
+            if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) {
+              vmParameters += ' -Djava.security.manager=allow'
+            }
           }
         }
         copyright {

libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentContraints.java

@@ -19,6 +19,7 @@
  */
 @InternalApi
 public interface XContentContraints {
+    final String DEFAULT_CODEPOINT_LIMIT_PROPERTY = "opensearch.xcontent.codepoint.max";
     final String DEFAULT_MAX_STRING_LEN_PROPERTY = "opensearch.xcontent.string.length.max";
     final String DEFAULT_MAX_NAME_LEN_PROPERTY = "opensearch.xcontent.name.length.max";
     final String DEFAULT_MAX_DEPTH_PROPERTY = "opensearch.xcontent.depth.max";
@@ -32,4 +33,6 @@ public interface XContentContraints {
     final int DEFAULT_MAX_DEPTH = Integer.parseInt(
         System.getProperty(DEFAULT_MAX_DEPTH_PROPERTY, "1000" /* StreamReadConstraints.DEFAULT_MAX_DEPTH */)
     );
+
+    final int DEFAULT_CODEPOINT_LIMIT = Integer.parseInt(System.getProperty(DEFAULT_CODEPOINT_LIMIT_PROPERTY, "52428800" /* ~50 Mb */));
 }

libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java

@@ -38,6 +38,7 @@
 import com.fasterxml.jackson.core.StreamReadFeature;
 import com.fasterxml.jackson.core.StreamWriteConstraints;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactoryBuilder;
 
 import org.opensearch.common.xcontent.XContentContraints;
 import org.opensearch.common.xcontent.XContentType;
@@ -55,6 +56,8 @@
 import java.io.Reader;
 import java.util.Set;
 
+import org.yaml.snakeyaml.LoaderOptions;
+
 /**
  * A YAML based content implementation using Jackson.
  */
@@ -67,7 +70,9 @@ public static XContentBuilder contentBuilder() throws IOException {
     public static final YamlXContent yamlXContent;
 
     static {
-        yamlFactory = new YAMLFactory();
+        final LoaderOptions loaderOptions = new LoaderOptions();
+        loaderOptions.setCodePointLimit(DEFAULT_CODEPOINT_LIMIT);
+        yamlFactory = new YAMLFactoryBuilder(new YAMLFactory()).loaderOptions(loaderOptions).build();
         yamlFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
         yamlFactory.setStreamWriteConstraints(StreamWriteConstraints.builder().maxNestingDepth(DEFAULT_MAX_DEPTH).build());
         yamlFactory.setStreamReadConstraints(

libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java

@@ -85,7 +85,7 @@ public class XContentParserTests extends OpenSearchTestCase {
         () -> randomAlphaOfLengthBetween(1, SmileXContent.DEFAULT_MAX_STRING_LEN),
         /* YAML parser limitation */
         XContentType.YAML,
-        () -> randomAlphaOfLengthBetween(1, 3140000)
+        () -> randomRealisticUnicodeOfCodepointLengthBetween(1, YamlXContent.DEFAULT_CODEPOINT_LIMIT)
     );
 
     private static final Map<XContentType, Supplier<String>> OFF_LIMIT_GENERATORS = Map.of(
@@ -97,7 +97,7 @@ public class XContentParserTests extends OpenSearchTestCase {
         () -> randomAlphaOfLength(SmileXContent.DEFAULT_MAX_STRING_LEN + 1),
         /* YAML parser limitation */
         XContentType.YAML,
-        () -> randomRealisticUnicodeOfCodepointLength(3145730)
+        () -> randomRealisticUnicodeOfCodepointLength(YamlXContent.DEFAULT_CODEPOINT_LIMIT + 1)
     );
 
     private static final Map<XContentType, Supplier<String>> FIELD_NAME_GENERATORS = Map.of(

plugins/identity-shiro/build.gradle

@@ -28,7 +28,7 @@ dependencies {
 
   implementation 'org.passay:passay:1.6.3'
 
-  implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
+  implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}"
 
   testImplementation project(path: ':modules:transport-netty4') // for http
   testImplementation project(path: ':plugins:transport-nio') // for http

plugins/identity-shiro/licenses/bcprov-jdk18on-1.77.jar.sha1

@@ -0,0 +1 @@
+2cc971b6c20949c1ff98d1a4bc741ee848a09523

plugins/ingest-attachment/build.gradle

@@ -71,9 +71,9 @@ dependencies {
   api "org.apache.pdfbox:fontbox:${versions.pdfbox}"
   api "org.apache.pdfbox:jempbox:1.8.17"
   api "commons-logging:commons-logging:${versions.commonslogging}"
-  api "org.bouncycastle:bcmail-jdk15to18:${versions.bouncycastle}"
-  api "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
-  api "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}"
+  api "org.bouncycastle:bcmail-jdk18on:${versions.bouncycastle}"
+  api "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}"
+  api "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
   // OpenOffice
   api "org.apache.poi:poi-ooxml:${versions.poi}"
   api "org.apache.poi:poi:${versions.poi}"

plugins/ingest-attachment/licenses/bcmail-jdk18on-1.77.jar.sha1

@@ -0,0 +1 @@
+f2bb8aa55dc901ee8b8aae7d1007c03592d65e03

plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.77.jar.sha1

@@ -0,0 +1 @@
+ed953791ba0229747dd0fd9911e3d76a462acfd3

plugins/ingest-attachment/licenses/bcprov-jdk18on-1.77.jar.sha1

@@ -0,0 +1 @@
+2cc971b6c20949c1ff98d1a4bc741ee848a09523

release-notes/opensearch.release-notes-2.12.0.md

@@ -128,7 +128,6 @@
 - Change error message when per shard document limit is breached ([#11312](https://github.com/opensearch-project/OpenSearch/pull/11312))
 - Improve boolean parsing performance ([#11308](https://github.com/opensearch-project/OpenSearch/pull/11308))
 - Interpret byte array as primitive using VarHandles ([#11362](https://github.com/opensearch-project/OpenSearch/pull/11362))
-- Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499))
 - Automatically add scheme to discovery.ec2.endpoint ([#11512](https://github.com/opensearch-project/OpenSearch/pull/11512))
 - Restore support for Java 8 for RestClient ([#11562](https://github.com/opensearch-project/OpenSearch/pull/11562))
 - Add deleted doc count in _cat/shards ([#11678](https://github.com/opensearch-project/OpenSearch/pull/11678))

server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java

@@ -59,19 +59,19 @@ public static Collection<Object[]> parameters() {
     @Override
     protected void setupSuiteScopeCluster() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("idx").get());
+        expected.clear();
 
-        final int segmentCount = randomIntBetween(2, 10);
-        final Set<Long> longTerms = new HashSet();
+        final int repeat = randomIntBetween(2, 10);
+        final Set<Long> longTerms = new HashSet<>();
 
-        final Map<String, Integer> dateTerms = new HashMap<>();
-        for (int i = 0; i < segmentCount; i++) {
+        for (int i = 0; i < repeat; i++) {
             final List<IndexRequestBuilder> indexRequests = new ArrayList<>();
 
             long longTerm;
             do {
-                longTerm = randomInt(segmentCount * 2);
+                longTerm = randomInt(repeat * 2);
             } while (!longTerms.add(longTerm));
-            ZonedDateTime time = ZonedDateTime.of(2024, 1, ((int) longTerm % 20) + 1, 0, 0, 0, 0, ZoneOffset.UTC);
+            ZonedDateTime time = ZonedDateTime.of(2024, 1, ((int) longTerm) + 1, 0, 0, 0, 0, ZoneOffset.UTC);
             String dateTerm = DateFormatter.forPattern("yyyy-MM-dd").format(time);
 
             final int frequency = randomBoolean() ? 1 : randomIntBetween(2, 20);

server/src/main/java/org/opensearch/common/lucene/store/ByteArrayIndexInput.java

@@ -144,7 +144,7 @@ public long readLong(long pos) throws IOException {
     }
 
     private void validatePos(long pos, int len) throws EOFException {
-        if (pos < 0 || pos + len > length + offset) {
+        if (pos < 0 || pos + len > length) {
             throw new EOFException("seek past EOF");
         }
     }

server/src/test/java/org/opensearch/common/lucene/store/ByteArrayIndexInputTests.java

@@ -32,6 +32,8 @@
 
 package org.opensearch.common.lucene.store;
 
+import org.apache.lucene.store.IndexInput;
+
 import java.io.EOFException;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -153,4 +155,34 @@ public void testRandomAccessReads() throws IOException {
         // 10001001 00100101 10001001 00110000 11100111 00100100 10110001 00101110
         assertEquals(-8564288273245753042L, indexInput.readLong(1));
     }
+
+    public void testReadBytesWithSlice() throws IOException {
+        int inputLength = randomIntBetween(100, 1000);
+
+        byte[] input = randomUnicodeOfLength(inputLength).getBytes(StandardCharsets.UTF_8);
+        ByteArrayIndexInput indexInput = new ByteArrayIndexInput("test", input);
+
+        int sliceOffset = randomIntBetween(1, inputLength - 10);
+        int sliceLength = randomIntBetween(2, inputLength - sliceOffset);
+        IndexInput slice = indexInput.slice("slice", sliceOffset, sliceLength);
+
+        // read a byte from sliced index input and verify if the read value is correct
+        assertEquals(input[sliceOffset], slice.readByte());
+
+        // read few more bytes into a byte array
+        int bytesToRead = randomIntBetween(1, sliceLength - 1);
+        slice.readBytes(new byte[bytesToRead], 0, bytesToRead);
+
+        // now try to read beyond the boundary of the slice, but within the
+        // boundary of the original IndexInput. We've already read few bytes
+        // so this is expected to fail
+        assertThrows(EOFException.class, () -> slice.readBytes(new byte[sliceLength], 0, sliceLength));
+
+        // seek to EOF and then try to read
+        slice.seek(sliceLength);
+        assertThrows(EOFException.class, () -> slice.readBytes(new byte[1], 0, 1));
+
+        slice.close();
+        indexInput.close();
+    }
 }

test/fixtures/hdfs-fixture/build.gradle

@@ -78,6 +78,6 @@ dependencies {
   runtimeOnly("com.squareup.okhttp3:okhttp:4.12.0") {
     exclude group: "com.squareup.okio"
   }
-  runtimeOnly "com.squareup.okio:okio:3.7.0"
+  runtimeOnly "com.squareup.okio:okio:3.8.0"
   runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.5"
 }