Merge branch 'main' into IT-PR

Signed-off-by: Ruirui Zhang <[email protected]>

CHANGELOG.md

@@ -43,7 +43,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133))
 - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135))
 - Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182))
+- Bump `com.google.api-client:google-api-client` from 2.2.0 to 2.7.0 ([#16216](https://github.com/opensearch-project/OpenSearch/pull/16216))
 - Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217))
+- Bump `io.grpc:grpc-api` from 1.57.2 to 1.68.0 ([#16213](https://github.com/opensearch-project/OpenSearch/pull/16213))
+- Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212))
 
 ### Changed
 - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049))
@@ -66,6 +69,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521))
 - Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158))
 - Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194))
+- Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254))
 
 ### Security
 

buildSrc/version.properties

@@ -25,6 +25,7 @@ jakarta_annotation = 1.3.5
 google_http_client = 1.44.1
 tdigest           = 3.3
 hdrhistogram      = 2.2.2
+grpc              = 1.68.0
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0

client/rest-high-level/build.gradle

@@ -50,7 +50,9 @@ restResources {
 }
 
 dependencies {
-  api project(':server')
+  api(project(':server')) {
+    exclude group: 'com.google.protobuf'
+  }
   api project(':client:rest')
   api project(':modules:mapper-extras')
   api project(':modules:parent-join')

libs/core/src/main/java/org/opensearch/Version.java

@@ -111,7 +111,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1);
-    public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1);
+    public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_12_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0);
     public static final Version CURRENT = V_3_0_0;
 

plugins/discovery-gce/build.gradle

@@ -30,7 +30,7 @@ dependencies {
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'io.grpc:grpc-api:1.57.2'
+  api "io.grpc:grpc-api:${versions.grpc}"
   api 'io.opencensus:opencensus-api:0.31.1'
   api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
   runtimeOnly "com.google.guava:guava:${versions.guava}"

plugins/discovery-gce/licenses/grpc-api-1.68.0.jar.sha1

@@ -0,0 +1 @@
+9a9f25c58d8d5b0fcf37ae889a50fec87e34ac08

plugins/repository-gcs/build.gradle

@@ -58,7 +58,7 @@ dependencies {
 
   api 'com.google.apis:google-api-services-storage:v1-rev20230617-2.0.0'
 
-  api 'com.google.api-client:google-api-client:2.2.0'
+  api 'com.google.api-client:google-api-client:2.7.0'
 
   api 'com.google.api.grpc:proto-google-common-protos:2.37.1'
   api 'com.google.api.grpc:proto-google-iam-v1:1.33.0'
@@ -86,7 +86,7 @@ dependencies {
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api 'org.threeten:threetenbp:1.4.4'
-  api 'io.grpc:grpc-api:1.57.2'
+  api "io.grpc:grpc-api:${versions.grpc}"
   api 'io.opencensus:opencensus-api:0.31.1'
   api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
 

plugins/repository-gcs/licenses/google-api-client-2.7.0.jar.sha1

@@ -0,0 +1 @@
+59c8e5e3c03f146561a83051af3ca945d40e02c6

plugins/repository-gcs/licenses/grpc-api-1.68.0.jar.sha1

@@ -0,0 +1 @@
+9a9f25c58d8d5b0fcf37ae889a50fec87e34ac08

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml

@@ -678,11 +678,6 @@ setup:
           - '{"index": {}}'
           - '{"date": "2016-03-01"}'
 
-  - do:
-      indices.forcemerge:
-        index: test_2
-        max_num_segments: 1
-
   - do:
       search:
         index: test_2

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml

@@ -1102,11 +1102,6 @@ setup:
           - '{"index": {}}'
           - '{"date": "2016-03-01"}'
 
-  - do:
-      indices.forcemerge:
-        index: test_2
-        max_num_segments: 1
-
   - do:
       search:
         index: test_2

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml

@@ -133,11 +133,6 @@ setup:
           - '{"index": {}}'
           - '{"date": "2020-03-09", "v": 4}'
 
-  - do:
-      indices.forcemerge:
-        index: test_profile
-        max_num_segments: 1
-
   - do:
       search:
         index: test_profile

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml

@@ -140,11 +140,6 @@ setup:
           - '{"index": {}}'
           - '{"date": "2025-02-14"}'
 
-  - do:
-      indices.forcemerge:
-        index: dhisto-agg-w-query
-        max_num_segments: 1
-
   - do:
       search:
         index: dhisto-agg-w-query

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml

@@ -568,11 +568,6 @@ setup:
           - '{"index": {}}'
           - '{"double" : 50}'
 
-  - do:
-      indices.forcemerge:
-        index: test_profile
-        max_num_segments: 1
-
   - do:
       search:
         index: test_profile

server/src/main/java/org/opensearch/common/cache/Cache.java

@@ -116,13 +116,13 @@ public class Cache<K, V> {
     private RemovalListener<K, V> removalListener = notification -> {};
 
     private final int numberOfSegments;
-    private static final int NUMBER_OF_SEGMENTS_DEFAULT = 256;
+    public static final int NUMBER_OF_SEGMENTS = 256;
 
     Cache(final int numberOfSegments) {
         if (numberOfSegments != -1) {
             this.numberOfSegments = numberOfSegments;
         } else {
-            this.numberOfSegments = NUMBER_OF_SEGMENTS_DEFAULT;
+            this.numberOfSegments = NUMBER_OF_SEGMENTS;
         }
         this.segments = new CacheSegment[this.numberOfSegments];
         for (int i = 0; i < this.numberOfSegments; i++) {

server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java

@@ -13,7 +13,6 @@
 import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.codec.composite.composite912.Composite912Codec;
-import org.opensearch.index.codec.composite.composite99.Composite99Codec;
 import org.opensearch.index.mapper.MapperService;
 
 import java.util.HashMap;
@@ -33,7 +32,7 @@
 public class CompositeCodecFactory {
 
     // we can use this to track the latest composite codec
-    public static final String COMPOSITE_CODEC = Composite99Codec.COMPOSITE_INDEX_CODEC_NAME;
+    public static final String COMPOSITE_CODEC = Composite912Codec.COMPOSITE_INDEX_CODEC_NAME;
 
     public CompositeCodecFactory() {}
 

server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java

@@ -11,7 +11,7 @@
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducerWrapper;
 import org.apache.lucene.index.SegmentReadState;
-import org.opensearch.index.codec.composite.composite99.Composite99Codec;
+import org.opensearch.index.codec.composite.composite912.Composite912Codec;
 
 import java.io.IOException;
 
@@ -39,7 +39,7 @@ public static DocValuesProducer getDocValuesProducerForCompositeCodec(
     ) throws IOException {
 
         switch (compositeCodec) {
-            case Composite99Codec.COMPOSITE_INDEX_CODEC_NAME:
+            case Composite912Codec.COMPOSITE_INDEX_CODEC_NAME:
                 Lucene90DocValuesProducerWrapper lucene90DocValuesProducerWrapper = new Lucene90DocValuesProducerWrapper(
                     state,
                     dataCodec,

server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912Codec.java

@@ -15,7 +15,6 @@
 import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
-import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat;
 import org.opensearch.index.mapper.MapperService;
 
 /**
@@ -53,6 +52,6 @@ protected Composite912Codec(String name, Codec delegate, MapperService mapperSer
 
     @Override
     public DocValuesFormat docValuesFormat() {
-        return new Composite99DocValuesFormat(mapperService);
+        return new Composite912DocValuesFormat(mapperService);
     }
 }

server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesFormat.java → server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesFormat.java

@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.index.codec.composite.composite99;
+package org.opensearch.index.codec.composite.composite912;
 
 import org.apache.lucene.codecs.DocValuesConsumer;
 import org.apache.lucene.codecs.DocValuesFormat;
@@ -25,7 +25,7 @@
  * @opensearch.experimental
  */
 @ExperimentalApi
-public class Composite99DocValuesFormat extends DocValuesFormat {
+public class Composite912DocValuesFormat extends DocValuesFormat {
     /**
      * Creates a new docvalues format.
      *
@@ -38,10 +38,10 @@ public class Composite99DocValuesFormat extends DocValuesFormat {
     private final MapperService mapperService;
 
     /** Data codec name for Composite Doc Values Format */
-    public static final String DATA_CODEC_NAME = "Composite99FormatData";
+    public static final String DATA_CODEC_NAME = "Composite912FormatData";
 
     /** Meta codec name for Composite Doc Values Format */
-    public static final String META_CODEC_NAME = "Composite99FormatMeta";
+    public static final String META_CODEC_NAME = "Composite912FormatMeta";
 
     /** Filename extension for the composite index data */
     public static final String DATA_EXTENSION = "cid";
@@ -50,10 +50,10 @@ public class Composite99DocValuesFormat extends DocValuesFormat {
     public static final String META_EXTENSION = "cim";
 
     /** Data doc values codec name for Composite Doc Values Format */
-    public static final String DATA_DOC_VALUES_CODEC = "Composite99DocValuesData";
+    public static final String DATA_DOC_VALUES_CODEC = "Composite912DocValuesData";
 
     /** Meta doc values codec name for Composite Doc Values Format */
-    public static final String META_DOC_VALUES_CODEC = "Composite99DocValuesMetadata";
+    public static final String META_DOC_VALUES_CODEC = "Composite912DocValuesMetadata";
 
     /** Filename extension for the composite index data doc values */
     public static final String DATA_DOC_VALUES_EXTENSION = "cidvd";
@@ -68,27 +68,27 @@ public class Composite99DocValuesFormat extends DocValuesFormat {
     public static final int VERSION_CURRENT = VERSION_START;
 
     // needed for SPI
-    public Composite99DocValuesFormat() {
+    public Composite912DocValuesFormat() {
         this(new Lucene90DocValuesFormat(), null);
     }
 
-    public Composite99DocValuesFormat(MapperService mapperService) {
+    public Composite912DocValuesFormat(MapperService mapperService) {
         this(new Lucene90DocValuesFormat(), mapperService);
     }
 
-    public Composite99DocValuesFormat(DocValuesFormat delegate, MapperService mapperService) {
+    public Composite912DocValuesFormat(DocValuesFormat delegate, MapperService mapperService) {
         super(delegate.getName());
         this.delegate = delegate;
         this.mapperService = mapperService;
     }
 
     @Override
     public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-        return new Composite99DocValuesWriter(delegate.fieldsConsumer(state), state, mapperService);
+        return new Composite912DocValuesWriter(delegate.fieldsConsumer(state), state, mapperService);
     }
 
     @Override
     public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
-        return new Composite99DocValuesReader(delegate.fieldsProducer(state), state);
+        return new Composite912DocValuesReader(delegate.fieldsProducer(state), state);
     }
 }

server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java → server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java

@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.index.codec.composite.composite99;
+package org.opensearch.index.codec.composite.composite912;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -56,8 +56,8 @@
  * @opensearch.experimental
  */
 @ExperimentalApi
-public class Composite99DocValuesReader extends DocValuesProducer implements CompositeIndexReader {
-    private static final Logger logger = LogManager.getLogger(Composite99DocValuesReader.class);
+public class Composite912DocValuesReader extends DocValuesProducer implements CompositeIndexReader {
+    private static final Logger logger = LogManager.getLogger(Composite912DocValuesReader.class);
 
     private final DocValuesProducer delegate;
     private IndexInput dataIn;
@@ -69,20 +69,20 @@ public class Composite99DocValuesReader extends DocValuesProducer implements Com
     private final List<CompositeIndexFieldInfo> compositeFieldInfos = new ArrayList<>();
     private SegmentReadState readState;
 
-    public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState readState) throws IOException {
+    public Composite912DocValuesReader(DocValuesProducer producer, SegmentReadState readState) throws IOException {
         this.delegate = producer;
         this.fields = new ArrayList<>();
 
         String metaFileName = IndexFileNames.segmentFileName(
             readState.segmentInfo.name,
             readState.segmentSuffix,
-            Composite99DocValuesFormat.META_EXTENSION
+            Composite912DocValuesFormat.META_EXTENSION
         );
 
         String dataFileName = IndexFileNames.segmentFileName(
             readState.segmentInfo.name,
             readState.segmentSuffix,
-            Composite99DocValuesFormat.DATA_EXTENSION
+            Composite912DocValuesFormat.DATA_EXTENSION
         );
 
         boolean success = false;
@@ -92,9 +92,9 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r
             dataIn = readState.directory.openInput(dataFileName, readState.context);
             CodecUtil.checkIndexHeader(
                 dataIn,
-                Composite99DocValuesFormat.DATA_CODEC_NAME,
-                Composite99DocValuesFormat.VERSION_START,
-                Composite99DocValuesFormat.VERSION_CURRENT,
+                Composite912DocValuesFormat.DATA_CODEC_NAME,
+                Composite912DocValuesFormat.VERSION_START,
+                Composite912DocValuesFormat.VERSION_CURRENT,
                 readState.segmentInfo.getId(),
                 readState.segmentSuffix
             );
@@ -105,9 +105,9 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r
             try {
                 CodecUtil.checkIndexHeader(
                     metaIn,
-                    Composite99DocValuesFormat.META_CODEC_NAME,
-                    Composite99DocValuesFormat.VERSION_START,
-                    Composite99DocValuesFormat.VERSION_CURRENT,
+                    Composite912DocValuesFormat.META_CODEC_NAME,
+                    Composite912DocValuesFormat.VERSION_START,
+                    Composite912DocValuesFormat.VERSION_CURRENT,
                     readState.segmentInfo.getId(),
                     readState.segmentSuffix
                 );
@@ -190,12 +190,12 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r
                 // initialize star-tree doc values producer
 
                 compositeDocValuesProducer = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec(
-                    Composite99Codec.COMPOSITE_INDEX_CODEC_NAME,
+                    Composite912Codec.COMPOSITE_INDEX_CODEC_NAME,
                     this.readState,
-                    Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC,
-                    Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION,
-                    Composite99DocValuesFormat.META_DOC_VALUES_CODEC,
-                    Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION
+                    Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC,
+                    Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION,
+                    Composite912DocValuesFormat.META_DOC_VALUES_CODEC,
+                    Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION
                 );
 
             } catch (Throwable t) {